drivers/net/ibm_newemac/core.c

   1 /*
   2  * drivers/net/ibm_newemac/core.c
   3  *
   4  * Driver for PowerPC 4xx on-chip ethernet controller.
   5  *
   6  * Copyright 2007 Benjamin Herrenschmidt, IBM Corp.
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * Based on the arch/ppc version of the driver:
  10  *
  11  * Copyright (c) 2004, 2005 Zultys Technologies.
  12  * Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>
  13  *
  14  * Based on original work by
  15  *      Matt Porter <mporter@kernel.crashing.org>
  16  *      (c) 2003 Benjamin Herrenschmidt <benh@kernel.crashing.org>
  17  *      Armin Kuster <akuster@mvista.com>
  18  *      Johnnie Peters <jpeters@mvista.com>
  19  *
  20  * This program is free software; you can redistribute  it and/or modify it
  21  * under  the terms of  the GNU General  Public License as published by the
  22  * Free Software Foundation;  either version 2 of the  License, or (at your
  23  * option) any later version.
  24  *
  25  */
  26
  27 #include <linux/sched.h>
  28 #include <linux/string.h>
  29 #include <linux/errno.h>
  30 #include <linux/delay.h>
  31 #include <linux/types.h>
  32 #include <linux/pci.h>
  33 #include <linux/etherdevice.h>
  34 #include <linux/skbuff.h>
  35 #include <linux/crc32.h>
  36 #include <linux/ethtool.h>
  37 #include <linux/mii.h>
  38 #include <linux/bitops.h>
  39 #include <linux/workqueue.h>
  40 #include <linux/of.h>
  41
  42 #include <asm/processor.h>
  43 #include <asm/io.h>
  44 #include <asm/dma.h>
  45 #include <asm/uaccess.h>
  46 #include <asm/dcr.h>
  47 #include <asm/dcr-regs.h>
  48
  49 #include "core.h"
  50
  51 /*
  52  * Lack of dma_unmap_???? calls is intentional.
  53  *
  54  * API-correct usage requires additional support state information to be
  55  * maintained for every RX and TX buffer descriptor (BD). Unfortunately, due to
  56  * EMAC design (e.g. TX buffer passed from network stack can be split into
  57  * several BDs, dma_map_single/dma_map_page can be used to map particular BD),
  58  * maintaining such information will add additional overhead.
  59  * Current DMA API implementation for 4xx processors only ensures cache coherency
  60  * and dma_unmap_???? routines are empty and are likely to stay this way.
  61  * I decided to omit dma_unmap_??? calls because I don't want to add additional
  62  * complexity just for the sake of following some abstract API, when it doesn't
  63  * add any real benefit to the driver. I understand that this decision maybe
  64  * controversial, but I really tried to make code API-correct and efficient
  65  * at the same time and didn't come up with code I liked :(.                --ebs
  66  */
  67
  68 #define DRV_NAME        "emac"
  69 #define DRV_VERSION     "3.54"
  70 #define DRV_DESC        "PPC 4xx OCP EMAC driver"
  71
  72 MODULE_DESCRIPTION(DRV_DESC);
  73 MODULE_AUTHOR
  74     ("Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>");
  75 MODULE_LICENSE("GPL");
  76
  77 /*
  78  * PPC64 doesn't (yet) have a cacheable_memcpy
  79  */
  80 #ifdef CONFIG_PPC64
  81 #define cacheable_memcpy(d,s,n) memcpy((d),(s),(n))
  82 #endif
  83
  84 /* minimum number of free TX descriptors required to wake up TX process */
  85 #define EMAC_TX_WAKEUP_THRESH           (NUM_TX_BUFF / 4)
  86
  87 /* If packet size is less than this number, we allocate small skb and copy packet
  88  * contents into it instead of just sending original big skb up
  89  */
  90 #define EMAC_RX_COPY_THRESH             CONFIG_IBM_NEW_EMAC_RX_COPY_THRESHOLD
  91
  92 /* Since multiple EMACs share MDIO lines in various ways, we need
  93  * to avoid re-using the same PHY ID in cases where the arch didn't
  94  * setup precise phy_map entries
  95  *
  96  * XXX This is something that needs to be reworked as we can have multiple
  97  * EMAC "sets" (multiple ASICs containing several EMACs) though we can
  98  * probably require in that case to have explicit PHY IDs in the device-tree
  99  */
 100 static u32 busy_phy_map;
 101 static DEFINE_MUTEX(emac_phy_map_lock);
 102
 103 /* This is the wait queue used to wait on any event related to probe, that
 104  * is discovery of MALs, other EMACs, ZMII/RGMIIs, etc...
 105  */
 106 static DECLARE_WAIT_QUEUE_HEAD(emac_probe_wait);
 107
 108 /* Having stable interface names is a doomed idea. However, it would be nice
 109  * if we didn't have completely random interface names at boot too :-) It's
 110  * just a matter of making everybody's life easier. Since we are doing
 111  * threaded probing, it's a bit harder though. The base idea here is that
 112  * we make up a list of all emacs in the device-tree before we register the
 113  * driver. Every emac will then wait for the previous one in the list to
 114  * initialize before itself. We should also keep that list ordered by
 115  * cell_index.
 116  * That list is only 4 entries long, meaning that additional EMACs don't
 117  * get ordering guarantees unless EMAC_BOOT_LIST_SIZE is increased.
 118  */
 119
 120 #define EMAC_BOOT_LIST_SIZE     4
 121 static struct device_node *emac_boot_list[EMAC_BOOT_LIST_SIZE];
 122
 123 /* How long should I wait for dependent devices ? */
 124 #define EMAC_PROBE_DEP_TIMEOUT  (HZ * 5)
 125
 126 /* I don't want to litter system log with timeout errors
 127  * when we have brain-damaged PHY.
 128  */
 129 static inline void emac_report_timeout_error(struct emac_instance *dev,
 130                                              const char *error)
 131 {
 132         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX |
 133                                   EMAC_FTR_460EX_PHY_CLK_FIX |
 134                                   EMAC_FTR_440EP_PHY_CLK_FIX))
 135                 DBG(dev, "%s" NL, error);
 136         else if (net_ratelimit())
 137                 printk(KERN_ERR "%s: %s\n", dev->ndev->name, error);
 138 }
 139
 140 /* EMAC PHY clock workaround:
 141  * 440EP/440GR has more sane SDR0_MFR register implementation than 440GX,
 142  * which allows controlling each EMAC clock
 143  */
 144 static inline void emac_rx_clk_tx(struct emac_instance *dev)
 145 {
 146 #ifdef CONFIG_PPC_DCR_NATIVE
 147         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 148                 dcri_clrset(SDR0, SDR0_MFR,
 149                             0, SDR0_MFR_ECS >> dev->cell_index);
 150 #endif
 151 }
 152
 153 static inline void emac_rx_clk_default(struct emac_instance *dev)
 154 {
 155 #ifdef CONFIG_PPC_DCR_NATIVE
 156         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 157                 dcri_clrset(SDR0, SDR0_MFR,
 158                             SDR0_MFR_ECS >> dev->cell_index, 0);
 159 #endif
 160 }
 161
 162 /* PHY polling intervals */
 163 #define PHY_POLL_LINK_ON        HZ
 164 #define PHY_POLL_LINK_OFF       (HZ / 5)
 165
 166 /* Graceful stop timeouts in us.
 167  * We should allow up to 1 frame time (full-duplex, ignoring collisions)
 168  */
 169 #define STOP_TIMEOUT_10         1230
 170 #define STOP_TIMEOUT_100        124
 171 #define STOP_TIMEOUT_1000       13
 172 #define STOP_TIMEOUT_1000_JUMBO 73
 173
 174 static unsigned char default_mcast_addr[] = {
 175         0x01, 0x80, 0xC2, 0x00, 0x00, 0x01
 176 };
 177
 178 /* Please, keep in sync with struct ibm_emac_stats/ibm_emac_error_stats */
 179 static const char emac_stats_keys[EMAC_ETHTOOL_STATS_COUNT][ETH_GSTRING_LEN] = {
 180         "rx_packets", "rx_bytes", "tx_packets", "tx_bytes", "rx_packets_csum",
 181         "tx_packets_csum", "tx_undo", "rx_dropped_stack", "rx_dropped_oom",
 182         "rx_dropped_error", "rx_dropped_resize", "rx_dropped_mtu",
 183         "rx_stopped", "rx_bd_errors", "rx_bd_overrun", "rx_bd_bad_packet",
 184         "rx_bd_runt_packet", "rx_bd_short_event", "rx_bd_alignment_error",
 185         "rx_bd_bad_fcs", "rx_bd_packet_too_long", "rx_bd_out_of_range",
 186         "rx_bd_in_range", "rx_parity", "rx_fifo_overrun", "rx_overrun",
 187         "rx_bad_packet", "rx_runt_packet", "rx_short_event",
 188         "rx_alignment_error", "rx_bad_fcs", "rx_packet_too_long",
 189         "rx_out_of_range", "rx_in_range", "tx_dropped", "tx_bd_errors",
 190         "tx_bd_bad_fcs", "tx_bd_carrier_loss", "tx_bd_excessive_deferral",
 191         "tx_bd_excessive_collisions", "tx_bd_late_collision",
 192         "tx_bd_multple_collisions", "tx_bd_single_collision",
 193         "tx_bd_underrun", "tx_bd_sqe", "tx_parity", "tx_underrun", "tx_sqe",
 194         "tx_errors"
 195 };
 196
 197 static irqreturn_t emac_irq(int irq, void *dev_instance);
 198 static void emac_clean_tx_ring(struct emac_instance *dev);
 199 static void __emac_set_multicast_list(struct emac_instance *dev);
 200
 201 static inline int emac_phy_supports_gige(int phy_mode)
 202 {
 203         return  phy_mode == PHY_MODE_GMII ||
 204                 phy_mode == PHY_MODE_RGMII ||
 205                 phy_mode == PHY_MODE_SGMII ||
 206                 phy_mode == PHY_MODE_TBI ||
 207                 phy_mode == PHY_MODE_RTBI;
 208 }
 209
 210 static inline int emac_phy_gpcs(int phy_mode)
 211 {
 212         return  phy_mode == PHY_MODE_SGMII ||
 213                 phy_mode == PHY_MODE_TBI ||
 214                 phy_mode == PHY_MODE_RTBI;
 215 }
 216
 217 static inline void emac_tx_enable(struct emac_instance *dev)
 218 {
 219         struct emac_regs __iomem *p = dev->emacp;
 220         u32 r;
 221
 222         DBG(dev, "tx_enable" NL);
 223
 224         r = in_be32(&p->mr0);
 225         if (!(r & EMAC_MR0_TXE))
 226                 out_be32(&p->mr0, r | EMAC_MR0_TXE);
 227 }
 228
 229 static void emac_tx_disable(struct emac_instance *dev)
 230 {
 231         struct emac_regs __iomem *p = dev->emacp;
 232         u32 r;
 233
 234         DBG(dev, "tx_disable" NL);
 235
 236         r = in_be32(&p->mr0);
 237         if (r & EMAC_MR0_TXE) {
 238                 int n = dev->stop_timeout;
 239                 out_be32(&p->mr0, r & ~EMAC_MR0_TXE);
 240                 while (!(in_be32(&p->mr0) & EMAC_MR0_TXI) && n) {
 241                         udelay(1);
 242                         --n;
 243                 }
 244                 if (unlikely(!n))
 245                         emac_report_timeout_error(dev, "TX disable timeout");
 246         }
 247 }
 248
 249 static void emac_rx_enable(struct emac_instance *dev)
 250 {
 251         struct emac_regs __iomem *p = dev->emacp;
 252         u32 r;
 253
 254         if (unlikely(test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags)))
 255                 goto out;
 256
 257         DBG(dev, "rx_enable" NL);
 258
 259         r = in_be32(&p->mr0);
 260         if (!(r & EMAC_MR0_RXE)) {
 261                 if (unlikely(!(r & EMAC_MR0_RXI))) {
 262                         /* Wait if previous async disable is still in progress */
 263                         int n = dev->stop_timeout;
 264                         while (!(r = in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 265                                 udelay(1);
 266                                 --n;
 267                         }
 268                         if (unlikely(!n))
 269                                 emac_report_timeout_error(dev,
 270                                                           "RX disable timeout");
 271                 }
 272                 out_be32(&p->mr0, r | EMAC_MR0_RXE);
 273         }
 274  out:
 275         ;
 276 }
 277
 278 static void emac_rx_disable(struct emac_instance *dev)
 279 {
 280         struct emac_regs __iomem *p = dev->emacp;
 281         u32 r;
 282
 283         DBG(dev, "rx_disable" NL);
 284
 285         r = in_be32(&p->mr0);
 286         if (r & EMAC_MR0_RXE) {
 287                 int n = dev->stop_timeout;
 288                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 289                 while (!(in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 290                         udelay(1);
 291                         --n;
 292                 }
 293                 if (unlikely(!n))
 294                         emac_report_timeout_error(dev, "RX disable timeout");
 295         }
 296 }
 297
 298 static inline void emac_netif_stop(struct emac_instance *dev)
 299 {
 300         netif_tx_lock_bh(dev->ndev);
 301         netif_addr_lock(dev->ndev);
 302         dev->no_mcast = 1;
 303         netif_addr_unlock(dev->ndev);
 304         netif_tx_unlock_bh(dev->ndev);
 305         dev->ndev->trans_start = jiffies;       /* prevent tx timeout */
 306         mal_poll_disable(dev->mal, &dev->commac);
 307         netif_tx_disable(dev->ndev);
 308 }
 309
 310 static inline void emac_netif_start(struct emac_instance *dev)
 311 {
 312         netif_tx_lock_bh(dev->ndev);
 313         netif_addr_lock(dev->ndev);
 314         dev->no_mcast = 0;
 315         if (dev->mcast_pending && netif_running(dev->ndev))
 316                 __emac_set_multicast_list(dev);
 317         netif_addr_unlock(dev->ndev);
 318         netif_tx_unlock_bh(dev->ndev);
 319
 320         netif_wake_queue(dev->ndev);
 321
 322         /* NOTE: unconditional netif_wake_queue is only appropriate
 323          * so long as all callers are assured to have free tx slots
 324          * (taken from tg3... though the case where that is wrong is
 325          *  not terribly harmful)
 326          */
 327         mal_poll_enable(dev->mal, &dev->commac);
 328 }
 329
 330 static inline void emac_rx_disable_async(struct emac_instance *dev)
 331 {
 332         struct emac_regs __iomem *p = dev->emacp;
 333         u32 r;
 334
 335         DBG(dev, "rx_disable_async" NL);
 336
 337         r = in_be32(&p->mr0);
 338         if (r & EMAC_MR0_RXE)
 339                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 340 }
 341
 342 static int emac_reset(struct emac_instance *dev)
 343 {
 344         struct emac_regs __iomem *p = dev->emacp;
 345         int n = 20;
 346
 347         DBG(dev, "reset" NL);
 348
 349         if (!dev->reset_failed) {
 350                 /* 40x erratum suggests stopping RX channel before reset,
 351                  * we stop TX as well
 352                  */
 353                 emac_rx_disable(dev);
 354                 emac_tx_disable(dev);
 355         }
 356
 357 #ifdef CONFIG_PPC_DCR_NATIVE
 358         /* Enable internal clock source */
 359         if (emac_has_feature(dev, EMAC_FTR_460EX_PHY_CLK_FIX))
 360                 dcri_clrset(SDR0, SDR0_ETH_CFG,
 361                             0, SDR0_ETH_CFG_ECS << dev->cell_index);
 362 #endif
 363
 364         out_be32(&p->mr0, EMAC_MR0_SRST);
 365         while ((in_be32(&p->mr0) & EMAC_MR0_SRST) && n)
 366                 --n;
 367
 368 #ifdef CONFIG_PPC_DCR_NATIVE
 369          /* Enable external clock source */
 370         if (emac_has_feature(dev, EMAC_FTR_460EX_PHY_CLK_FIX))
 371                 dcri_clrset(SDR0, SDR0_ETH_CFG,
 372                             SDR0_ETH_CFG_ECS << dev->cell_index, 0);
 373 #endif
 374
 375         if (n) {
 376                 dev->reset_failed = 0;
 377                 return 0;
 378         } else {
 379                 emac_report_timeout_error(dev, "reset timeout");
 380                 dev->reset_failed = 1;
 381                 return -ETIMEDOUT;
 382         }
 383 }
 384
 385 static void emac_hash_mc(struct emac_instance *dev)
 386 {
 387         const int regs = EMAC_XAHT_REGS(dev);
 388         u32 *gaht_base = emac_gaht_base(dev);
 389         u32 gaht_temp[regs];
 390         struct dev_mc_list *dmi;
 391         int i;
 392
 393         DBG(dev, "hash_mc %d" NL, dev->ndev->mc_count);
 394
 395         memset(gaht_temp, 0, sizeof (gaht_temp));
 396
 397         for (dmi = dev->ndev->mc_list; dmi; dmi = dmi->next) {
 398                 int slot, reg, mask;
 399                 DBG2(dev, "mc %pM" NL, dmi->dmi_addr);
 400
 401                 slot = EMAC_XAHT_CRC_TO_SLOT(dev, ether_crc(ETH_ALEN, dmi->dmi_addr));
 402                 reg = EMAC_XAHT_SLOT_TO_REG(dev, slot);
 403                 mask = EMAC_XAHT_SLOT_TO_MASK(dev, slot);
 404
 405                 gaht_temp[reg] |= mask;
 406         }
 407
 408         for (i = 0; i < regs; i++)
 409                 out_be32(gaht_base + i, gaht_temp[i]);
 410 }
 411
 412 static inline u32 emac_iff2rmr(struct net_device *ndev)
 413 {
 414         struct emac_instance *dev = netdev_priv(ndev);
 415         u32 r;
 416
 417         r = EMAC_RMR_SP | EMAC_RMR_SFCS | EMAC_RMR_IAE | EMAC_RMR_BAE;
 418
 419         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 420             r |= EMAC4_RMR_BASE;
 421         else
 422             r |= EMAC_RMR_BASE;
 423
 424         if (ndev->flags & IFF_PROMISC)
 425                 r |= EMAC_RMR_PME;
 426         else if (ndev->flags & IFF_ALLMULTI ||
 427                          (ndev->mc_count > EMAC_XAHT_SLOTS(dev)))
 428                 r |= EMAC_RMR_PMME;
 429         else if (ndev->mc_count > 0)
 430                 r |= EMAC_RMR_MAE;
 431
 432         return r;
 433 }
 434
 435 static u32 __emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 436 {
 437         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC_MR1_TR0_MULT;
 438
 439         DBG2(dev, "__emac_calc_base_mr1" NL);
 440
 441         switch(tx_size) {
 442         case 2048:
 443                 ret |= EMAC_MR1_TFS_2K;
 444                 break;
 445         default:
 446                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 447                        dev->ndev->name, tx_size);
 448         }
 449
 450         switch(rx_size) {
 451         case 16384:
 452                 ret |= EMAC_MR1_RFS_16K;
 453                 break;
 454         case 4096:
 455                 ret |= EMAC_MR1_RFS_4K;
 456                 break;
 457         default:
 458                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 459                        dev->ndev->name, rx_size);
 460         }
 461
 462         return ret;
 463 }
 464
 465 static u32 __emac4_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 466 {
 467         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC4_MR1_TR |
 468                 EMAC4_MR1_OBCI(dev->opb_bus_freq / 1000000);
 469
 470         DBG2(dev, "__emac4_calc_base_mr1" NL);
 471
 472         switch(tx_size) {
 473         case 4096:
 474                 ret |= EMAC4_MR1_TFS_4K;
 475                 break;
 476         case 2048:
 477                 ret |= EMAC4_MR1_TFS_2K;
 478                 break;
 479         default:
 480                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 481                        dev->ndev->name, tx_size);
 482         }
 483
 484         switch(rx_size) {
 485         case 16384:
 486                 ret |= EMAC4_MR1_RFS_16K;
 487                 break;
 488         case 4096:
 489                 ret |= EMAC4_MR1_RFS_4K;
 490                 break;
 491         case 2048:
 492                 ret |= EMAC4_MR1_RFS_2K;
 493                 break;
 494         default:
 495                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 496                        dev->ndev->name, rx_size);
 497         }
 498
 499         return ret;
 500 }
 501
 502 static u32 emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 503 {
 504         return emac_has_feature(dev, EMAC_FTR_EMAC4) ?
 505                 __emac4_calc_base_mr1(dev, tx_size, rx_size) :
 506                 __emac_calc_base_mr1(dev, tx_size, rx_size);
 507 }
 508
 509 static inline u32 emac_calc_trtr(struct emac_instance *dev, unsigned int size)
 510 {
 511         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 512                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT_EMAC4;
 513         else
 514                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT;
 515 }
 516
 517 static inline u32 emac_calc_rwmr(struct emac_instance *dev,
 518                                  unsigned int low, unsigned int high)
 519 {
 520         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 521                 return (low << 22) | ( (high & 0x3ff) << 6);
 522         else
 523                 return (low << 23) | ( (high & 0x1ff) << 7);
 524 }
 525
 526 static int emac_configure(struct emac_instance *dev)
 527 {
 528         struct emac_regs __iomem *p = dev->emacp;
 529         struct net_device *ndev = dev->ndev;
 530         int tx_size, rx_size, link = netif_carrier_ok(dev->ndev);
 531         u32 r, mr1 = 0;
 532
 533         DBG(dev, "configure" NL);
 534
 535         if (!link) {
 536                 out_be32(&p->mr1, in_be32(&p->mr1)
 537                          | EMAC_MR1_FDE | EMAC_MR1_ILE);
 538                 udelay(100);
 539         } else if (emac_reset(dev) < 0)
 540                 return -ETIMEDOUT;
 541
 542         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
 543                 tah_reset(dev->tah_dev);
 544
 545         DBG(dev, " link = %d duplex = %d, pause = %d, asym_pause = %d\n",
 546             link, dev->phy.duplex, dev->phy.pause, dev->phy.asym_pause);
 547
 548         /* Default fifo sizes */
 549         tx_size = dev->tx_fifo_size;
 550         rx_size = dev->rx_fifo_size;
 551
 552         /* No link, force loopback */
 553         if (!link)
 554                 mr1 = EMAC_MR1_FDE | EMAC_MR1_ILE;
 555
 556         /* Check for full duplex */
 557         else if (dev->phy.duplex == DUPLEX_FULL)
 558                 mr1 |= EMAC_MR1_FDE | EMAC_MR1_MWSW_001;
 559
 560         /* Adjust fifo sizes, mr1 and timeouts based on link speed */
 561         dev->stop_timeout = STOP_TIMEOUT_10;
 562         switch (dev->phy.speed) {
 563         case SPEED_1000:
 564                 if (emac_phy_gpcs(dev->phy.mode)) {
 565                         mr1 |= EMAC_MR1_MF_1000GPCS | EMAC_MR1_MF_IPPA(
 566                                 (dev->phy.gpcs_address != 0xffffffff) ?
 567                                  dev->phy.gpcs_address : dev->phy.address);
 568
 569                         /* Put some arbitrary OUI, Manuf & Rev IDs so we can
 570                          * identify this GPCS PHY later.
 571                          */
 572                         out_be32(&p->u1.emac4.ipcr, 0xdeadbeef);
 573                 } else
 574                         mr1 |= EMAC_MR1_MF_1000;
 575
 576                 /* Extended fifo sizes */
 577                 tx_size = dev->tx_fifo_size_gige;
 578                 rx_size = dev->rx_fifo_size_gige;
 579
 580                 if (dev->ndev->mtu > ETH_DATA_LEN) {
 581                         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 582                                 mr1 |= EMAC4_MR1_JPSM;
 583                         else
 584                                 mr1 |= EMAC_MR1_JPSM;
 585                         dev->stop_timeout = STOP_TIMEOUT_1000_JUMBO;
 586                 } else
 587                         dev->stop_timeout = STOP_TIMEOUT_1000;
 588                 break;
 589         case SPEED_100:
 590                 mr1 |= EMAC_MR1_MF_100;
 591                 dev->stop_timeout = STOP_TIMEOUT_100;
 592                 break;
 593         default: /* make gcc happy */
 594                 break;
 595         }
 596
 597         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 598                 rgmii_set_speed(dev->rgmii_dev, dev->rgmii_port,
 599                                 dev->phy.speed);
 600         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 601                 zmii_set_speed(dev->zmii_dev, dev->zmii_port, dev->phy.speed);
 602
 603         /* on 40x erratum forces us to NOT use integrated flow control,
 604          * let's hope it works on 44x ;)
 605          */
 606         if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x) &&
 607             dev->phy.duplex == DUPLEX_FULL) {
 608                 if (dev->phy.pause)
 609                         mr1 |= EMAC_MR1_EIFC | EMAC_MR1_APP;
 610                 else if (dev->phy.asym_pause)
 611                         mr1 |= EMAC_MR1_APP;
 612         }
 613
 614         /* Add base settings & fifo sizes & program MR1 */
 615         mr1 |= emac_calc_base_mr1(dev, tx_size, rx_size);
 616         out_be32(&p->mr1, mr1);
 617
 618         /* Set individual MAC address */
 619         out_be32(&p->iahr, (ndev->dev_addr[0] << 8) | ndev->dev_addr[1]);
 620         out_be32(&p->ialr, (ndev->dev_addr[2] << 24) |
 621                  (ndev->dev_addr[3] << 16) | (ndev->dev_addr[4] << 8) |
 622                  ndev->dev_addr[5]);
 623
 624         /* VLAN Tag Protocol ID */
 625         out_be32(&p->vtpid, 0x8100);
 626
 627         /* Receive mode register */
 628         r = emac_iff2rmr(ndev);
 629         if (r & EMAC_RMR_MAE)
 630                 emac_hash_mc(dev);
 631         out_be32(&p->rmr, r);
 632
 633         /* FIFOs thresholds */
 634         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 635                 r = EMAC4_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 636                                tx_size / 2 / dev->fifo_entry_size);
 637         else
 638                 r = EMAC_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 639                               tx_size / 2 / dev->fifo_entry_size);
 640         out_be32(&p->tmr1, r);
 641         out_be32(&p->trtr, emac_calc_trtr(dev, tx_size / 2));
 642
 643         /* PAUSE frame is sent when RX FIFO reaches its high-water mark,
 644            there should be still enough space in FIFO to allow the our link
 645            partner time to process this frame and also time to send PAUSE
 646            frame itself.
 647
 648            Here is the worst case scenario for the RX FIFO "headroom"
 649            (from "The Switch Book") (100Mbps, without preamble, inter-frame gap):
 650
 651            1) One maximum-length frame on TX                    1522 bytes
 652            2) One PAUSE frame time                                64 bytes
 653            3) PAUSE frame decode time allowance                   64 bytes
 654            4) One maximum-length frame on RX                    1522 bytes
 655            5) Round-trip propagation delay of the link (100Mb)    15 bytes
 656            ----------
 657            3187 bytes
 658
 659            I chose to set high-water mark to RX_FIFO_SIZE / 4 (1024 bytes)
 660            low-water mark  to RX_FIFO_SIZE / 8 (512 bytes)
 661          */
 662         r = emac_calc_rwmr(dev, rx_size / 8 / dev->fifo_entry_size,
 663                            rx_size / 4 / dev->fifo_entry_size);
 664         out_be32(&p->rwmr, r);
 665
 666         /* Set PAUSE timer to the maximum */
 667         out_be32(&p->ptr, 0xffff);
 668
 669         /* IRQ sources */
 670         r = EMAC_ISR_OVR | EMAC_ISR_BP | EMAC_ISR_SE |
 671                 EMAC_ISR_ALE | EMAC_ISR_BFCS | EMAC_ISR_PTLE | EMAC_ISR_ORE |
 672                 EMAC_ISR_IRE | EMAC_ISR_TE;
 673         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 674             r |= EMAC4_ISR_TXPE | EMAC4_ISR_RXPE /* | EMAC4_ISR_TXUE |
 675                                                   EMAC4_ISR_RXOE | */;
 676         out_be32(&p->iser,  r);
 677
 678         /* We need to take GPCS PHY out of isolate mode after EMAC reset */
 679         if (emac_phy_gpcs(dev->phy.mode)) {
 680                 if (dev->phy.gpcs_address != 0xffffffff)
 681                         emac_mii_reset_gpcs(&dev->phy);
 682                 else
 683                         emac_mii_reset_phy(&dev->phy);
 684         }
 685
 686         return 0;
 687 }
 688
 689 static void emac_reinitialize(struct emac_instance *dev)
 690 {
 691         DBG(dev, "reinitialize" NL);
 692
 693         emac_netif_stop(dev);
 694         if (!emac_configure(dev)) {
 695                 emac_tx_enable(dev);
 696                 emac_rx_enable(dev);
 697         }
 698         emac_netif_start(dev);
 699 }
 700
 701 static void emac_full_tx_reset(struct emac_instance *dev)
 702 {
 703         DBG(dev, "full_tx_reset" NL);
 704
 705         emac_tx_disable(dev);
 706         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
 707         emac_clean_tx_ring(dev);
 708         dev->tx_cnt = dev->tx_slot = dev->ack_slot = 0;
 709
 710         emac_configure(dev);
 711
 712         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
 713         emac_tx_enable(dev);
 714         emac_rx_enable(dev);
 715 }
 716
 717 static void emac_reset_work(struct work_struct *work)
 718 {
 719         struct emac_instance *dev = container_of(work, struct emac_instance, reset_work);
 720
 721         DBG(dev, "reset_work" NL);
 722
 723         mutex_lock(&dev->link_lock);
 724         if (dev->opened) {
 725                 emac_netif_stop(dev);
 726                 emac_full_tx_reset(dev);
 727                 emac_netif_start(dev);
 728         }
 729         mutex_unlock(&dev->link_lock);
 730 }
 731
 732 static void emac_tx_timeout(struct net_device *ndev)
 733 {
 734         struct emac_instance *dev = netdev_priv(ndev);
 735
 736         DBG(dev, "tx_timeout" NL);
 737
 738         schedule_work(&dev->reset_work);
 739 }
 740
 741
 742 static inline int emac_phy_done(struct emac_instance *dev, u32 stacr)
 743 {
 744         int done = !!(stacr & EMAC_STACR_OC);
 745
 746         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 747                 done = !done;
 748
 749         return done;
 750 };
 751
 752 static int __emac_mdio_read(struct emac_instance *dev, u8 id, u8 reg)
 753 {
 754         struct emac_regs __iomem *p = dev->emacp;
 755         u32 r = 0;
 756         int n, err = -ETIMEDOUT;
 757
 758         mutex_lock(&dev->mdio_lock);
 759
 760         DBG2(dev, "mdio_read(%02x,%02x)" NL, id, reg);
 761
 762         /* Enable proper MDIO port */
 763         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 764                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 765         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 766                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 767
 768         /* Wait for management interface to become idle */
 769         n = 20;
 770         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 771                 udelay(1);
 772                 if (!--n) {
 773                         DBG2(dev, " -> timeout wait idle\n");
 774                         goto bail;
 775                 }
 776         }
 777
 778         /* Issue read command */
 779         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 780                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 781         else
 782                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 783         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 784                 r |= EMAC_STACR_OC;
 785         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 786                 r |= EMACX_STACR_STAC_READ;
 787         else
 788                 r |= EMAC_STACR_STAC_READ;
 789         r |= (reg & EMAC_STACR_PRA_MASK)
 790                 | ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT);
 791         out_be32(&p->stacr, r);
 792
 793         /* Wait for read to complete */
 794         n = 200;
 795         while (!emac_phy_done(dev, (r = in_be32(&p->stacr)))) {
 796                 udelay(1);
 797                 if (!--n) {
 798                         DBG2(dev, " -> timeout wait complete\n");
 799                         goto bail;
 800                 }
 801         }
 802
 803         if (unlikely(r & EMAC_STACR_PHYE)) {
 804                 DBG(dev, "mdio_read(%02x, %02x) failed" NL, id, reg);
 805                 err = -EREMOTEIO;
 806                 goto bail;
 807         }
 808
 809         r = ((r >> EMAC_STACR_PHYD_SHIFT) & EMAC_STACR_PHYD_MASK);
 810
 811         DBG2(dev, "mdio_read -> %04x" NL, r);
 812         err = 0;
 813  bail:
 814         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 815                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 816         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 817                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 818         mutex_unlock(&dev->mdio_lock);
 819
 820         return err == 0 ? r : err;
 821 }
 822
 823 static void __emac_mdio_write(struct emac_instance *dev, u8 id, u8 reg,
 824                               u16 val)
 825 {
 826         struct emac_regs __iomem *p = dev->emacp;
 827         u32 r = 0;
 828         int n, err = -ETIMEDOUT;
 829
 830         mutex_lock(&dev->mdio_lock);
 831
 832         DBG2(dev, "mdio_write(%02x,%02x,%04x)" NL, id, reg, val);
 833
 834         /* Enable proper MDIO port */
 835         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 836                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 837         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 838                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 839
 840         /* Wait for management interface to be idle */
 841         n = 20;
 842         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 843                 udelay(1);
 844                 if (!--n) {
 845                         DBG2(dev, " -> timeout wait idle\n");
 846                         goto bail;
 847                 }
 848         }
 849
 850         /* Issue write command */
 851         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 852                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 853         else
 854                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 855         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 856                 r |= EMAC_STACR_OC;
 857         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 858                 r |= EMACX_STACR_STAC_WRITE;
 859         else
 860                 r |= EMAC_STACR_STAC_WRITE;
 861         r |= (reg & EMAC_STACR_PRA_MASK) |
 862                 ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT) |
 863                 (val << EMAC_STACR_PHYD_SHIFT);
 864         out_be32(&p->stacr, r);
 865
 866         /* Wait for write to complete */
 867         n = 200;
 868         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 869                 udelay(1);
 870                 if (!--n) {
 871                         DBG2(dev, " -> timeout wait complete\n");
 872                         goto bail;
 873                 }
 874         }
 875         err = 0;
 876  bail:
 877         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 878                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 879         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 880                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 881         mutex_unlock(&dev->mdio_lock);
 882 }
 883
 884 static int emac_mdio_read(struct net_device *ndev, int id, int reg)
 885 {
 886         struct emac_instance *dev = netdev_priv(ndev);
 887         int res;
 888
 889         res = __emac_mdio_read((dev->mdio_instance &&
 890                                 dev->phy.gpcs_address != id) ?
 891                                 dev->mdio_instance : dev,
 892                                (u8) id, (u8) reg);
 893         return res;
 894 }
 895
 896 static void emac_mdio_write(struct net_device *ndev, int id, int reg, int val)
 897 {
 898         struct emac_instance *dev = netdev_priv(ndev);
 899
 900         __emac_mdio_write((dev->mdio_instance &&
 901                            dev->phy.gpcs_address != id) ?
 902                            dev->mdio_instance : dev,
 903                           (u8) id, (u8) reg, (u16) val);
 904 }
 905
 906 /* Tx lock BH */
 907 static void __emac_set_multicast_list(struct emac_instance *dev)
 908 {
 909         struct emac_regs __iomem *p = dev->emacp;
 910         u32 rmr = emac_iff2rmr(dev->ndev);
 911
 912         DBG(dev, "__multicast %08x" NL, rmr);
 913
 914         /* I decided to relax register access rules here to avoid
 915          * full EMAC reset.
 916          *
 917          * There is a real problem with EMAC4 core if we use MWSW_001 bit
 918          * in MR1 register and do a full EMAC reset.
 919          * One TX BD status update is delayed and, after EMAC reset, it
 920          * never happens, resulting in TX hung (it'll be recovered by TX
 921          * timeout handler eventually, but this is just gross).
 922          * So we either have to do full TX reset or try to cheat here :)
 923          *
 924          * The only required change is to RX mode register, so I *think* all
 925          * we need is just to stop RX channel. This seems to work on all
 926          * tested SoCs.                                                --ebs
 927          *
 928          * If we need the full reset, we might just trigger the workqueue
 929          * and do it async... a bit nasty but should work --BenH
 930          */
 931         dev->mcast_pending = 0;
 932         emac_rx_disable(dev);
 933         if (rmr & EMAC_RMR_MAE)
 934                 emac_hash_mc(dev);
 935         out_be32(&p->rmr, rmr);
 936         emac_rx_enable(dev);
 937 }
 938
 939 /* Tx lock BH */
 940 static void emac_set_multicast_list(struct net_device *ndev)
 941 {
 942         struct emac_instance *dev = netdev_priv(ndev);
 943
 944         DBG(dev, "multicast" NL);
 945
 946         BUG_ON(!netif_running(dev->ndev));
 947
 948         if (dev->no_mcast) {
 949                 dev->mcast_pending = 1;
 950                 return;
 951         }
 952         __emac_set_multicast_list(dev);
 953 }
 954
 955 static int emac_resize_rx_ring(struct emac_instance *dev, int new_mtu)
 956 {
 957         int rx_sync_size = emac_rx_sync_size(new_mtu);
 958         int rx_skb_size = emac_rx_skb_size(new_mtu);
 959         int i, ret = 0;
 960
 961         mutex_lock(&dev->link_lock);
 962         emac_netif_stop(dev);
 963         emac_rx_disable(dev);
 964         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
 965
 966         if (dev->rx_sg_skb) {
 967                 ++dev->estats.rx_dropped_resize;
 968                 dev_kfree_skb(dev->rx_sg_skb);
 969                 dev->rx_sg_skb = NULL;
 970         }
 971
 972         /* Make a first pass over RX ring and mark BDs ready, dropping
 973          * non-processed packets on the way. We need this as a separate pass
 974          * to simplify error recovery in the case of allocation failure later.
 975          */
 976         for (i = 0; i < NUM_RX_BUFF; ++i) {
 977                 if (dev->rx_desc[i].ctrl & MAL_RX_CTRL_FIRST)
 978                         ++dev->estats.rx_dropped_resize;
 979
 980                 dev->rx_desc[i].data_len = 0;
 981                 dev->rx_desc[i].ctrl = MAL_RX_CTRL_EMPTY |
 982                     (i == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
 983         }
 984
 985         /* Reallocate RX ring only if bigger skb buffers are required */
 986         if (rx_skb_size <= dev->rx_skb_size)
 987                 goto skip;
 988
 989         /* Second pass, allocate new skbs */
 990         for (i = 0; i < NUM_RX_BUFF; ++i) {
 991                 struct sk_buff *skb = alloc_skb(rx_skb_size, GFP_ATOMIC);
 992                 if (!skb) {
 993                         ret = -ENOMEM;
 994                         goto oom;
 995                 }
 996
 997                 BUG_ON(!dev->rx_skb[i]);
 998                 dev_kfree_skb(dev->rx_skb[i]);
 999
1000                 skb_reserve(skb, EMAC_RX_SKB_HEADROOM + 2);
1001                 dev->rx_desc[i].data_ptr =
1002                     dma_map_single(&dev->ofdev->dev, skb->data - 2, rx_sync_size,
1003                                    DMA_FROM_DEVICE) + 2;
1004                 dev->rx_skb[i] = skb;
1005         }
1006  skip:
1007         /* Check if we need to change "Jumbo" bit in MR1 */
1008         if ((new_mtu > ETH_DATA_LEN) ^ (dev->ndev->mtu > ETH_DATA_LEN)) {
1009                 /* This is to prevent starting RX channel in emac_rx_enable() */
1010                 set_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1011
1012                 dev->ndev->mtu = new_mtu;
1013                 emac_full_tx_reset(dev);
1014         }
1015
1016         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(new_mtu));
1017  oom:
1018         /* Restart RX */
1019         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1020         dev->rx_slot = 0;
1021         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1022         emac_rx_enable(dev);
1023         emac_netif_start(dev);
1024         mutex_unlock(&dev->link_lock);
1025
1026         return ret;
1027 }
1028
1029 /* Process ctx, rtnl_lock semaphore */
1030 static int emac_change_mtu(struct net_device *ndev, int new_mtu)
1031 {
1032         struct emac_instance *dev = netdev_priv(ndev);
1033         int ret = 0;
1034
1035         if (new_mtu < EMAC_MIN_MTU || new_mtu > dev->max_mtu)
1036                 return -EINVAL;
1037
1038         DBG(dev, "change_mtu(%d)" NL, new_mtu);
1039
1040         if (netif_running(ndev)) {
1041                 /* Check if we really need to reinitalize RX ring */
1042                 if (emac_rx_skb_size(ndev->mtu) != emac_rx_skb_size(new_mtu))
1043                         ret = emac_resize_rx_ring(dev, new_mtu);
1044         }
1045
1046         if (!ret) {
1047                 ndev->mtu = new_mtu;
1048                 dev->rx_skb_size = emac_rx_skb_size(new_mtu);
1049                 dev->rx_sync_size = emac_rx_sync_size(new_mtu);
1050         }
1051
1052         return ret;
1053 }
1054
1055 static void emac_clean_tx_ring(struct emac_instance *dev)
1056 {
1057         int i;
1058
1059         for (i = 0; i < NUM_TX_BUFF; ++i) {
1060                 if (dev->tx_skb[i]) {
1061                         dev_kfree_skb(dev->tx_skb[i]);
1062                         dev->tx_skb[i] = NULL;
1063                         if (dev->tx_desc[i].ctrl & MAL_TX_CTRL_READY)
1064                                 ++dev->estats.tx_dropped;
1065                 }
1066                 dev->tx_desc[i].ctrl = 0;
1067                 dev->tx_desc[i].data_ptr = 0;
1068         }
1069 }
1070
1071 static void emac_clean_rx_ring(struct emac_instance *dev)
1072 {
1073         int i;
1074
1075         for (i = 0; i < NUM_RX_BUFF; ++i)
1076                 if (dev->rx_skb[i]) {
1077                         dev->rx_desc[i].ctrl = 0;
1078                         dev_kfree_skb(dev->rx_skb[i]);
1079                         dev->rx_skb[i] = NULL;
1080                         dev->rx_desc[i].data_ptr = 0;
1081                 }
1082
1083         if (dev->rx_sg_skb) {
1084                 dev_kfree_skb(dev->rx_sg_skb);
1085                 dev->rx_sg_skb = NULL;
1086         }
1087 }
1088
1089 static inline int emac_alloc_rx_skb(struct emac_instance *dev, int slot,
1090                                     gfp_t flags)
1091 {
1092         struct sk_buff *skb = alloc_skb(dev->rx_skb_size, flags);
1093         if (unlikely(!skb))
1094                 return -ENOMEM;
1095
1096         dev->rx_skb[slot] = skb;
1097         dev->rx_desc[slot].data_len = 0;
1098
1099         skb_reserve(skb, EMAC_RX_SKB_HEADROOM + 2);
1100         dev->rx_desc[slot].data_ptr =
1101             dma_map_single(&dev->ofdev->dev, skb->data - 2, dev->rx_sync_size,
1102                            DMA_FROM_DEVICE) + 2;
1103         wmb();
1104         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1105             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1106
1107         return 0;
1108 }
1109
1110 static void emac_print_link_status(struct emac_instance *dev)
1111 {
1112         if (netif_carrier_ok(dev->ndev))
1113                 printk(KERN_INFO "%s: link is up, %d %s%s\n",
1114                        dev->ndev->name, dev->phy.speed,
1115                        dev->phy.duplex == DUPLEX_FULL ? "FDX" : "HDX",
1116                        dev->phy.pause ? ", pause enabled" :
1117                        dev->phy.asym_pause ? ", asymmetric pause enabled" : "");
1118         else
1119                 printk(KERN_INFO "%s: link is down\n", dev->ndev->name);
1120 }
1121
1122 /* Process ctx, rtnl_lock semaphore */
1123 static int emac_open(struct net_device *ndev)
1124 {
1125         struct emac_instance *dev = netdev_priv(ndev);
1126         int err, i;
1127
1128         DBG(dev, "open" NL);
1129
1130         /* Setup error IRQ handler */
1131         err = request_irq(dev->emac_irq, emac_irq, 0, "EMAC", dev);
1132         if (err) {
1133                 printk(KERN_ERR "%s: failed to request IRQ %d\n",
1134                        ndev->name, dev->emac_irq);
1135                 return err;
1136         }
1137
1138         /* Allocate RX ring */
1139         for (i = 0; i < NUM_RX_BUFF; ++i)
1140                 if (emac_alloc_rx_skb(dev, i, GFP_KERNEL)) {
1141                         printk(KERN_ERR "%s: failed to allocate RX ring\n",
1142                                ndev->name);
1143                         goto oom;
1144                 }
1145
1146         dev->tx_cnt = dev->tx_slot = dev->ack_slot = dev->rx_slot = 0;
1147         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1148         dev->rx_sg_skb = NULL;
1149
1150         mutex_lock(&dev->link_lock);
1151         dev->opened = 1;
1152
1153         /* Start PHY polling now.
1154          */
1155         if (dev->phy.address >= 0) {
1156                 int link_poll_interval;
1157                 if (dev->phy.def->ops->poll_link(&dev->phy)) {
1158                         dev->phy.def->ops->read_link(&dev->phy);
1159                         emac_rx_clk_default(dev);
1160                         netif_carrier_on(dev->ndev);
1161                         link_poll_interval = PHY_POLL_LINK_ON;
1162                 } else {
1163                         emac_rx_clk_tx(dev);
1164                         netif_carrier_off(dev->ndev);
1165                         link_poll_interval = PHY_POLL_LINK_OFF;
1166                 }
1167                 dev->link_polling = 1;
1168                 wmb();
1169                 schedule_delayed_work(&dev->link_work, link_poll_interval);
1170                 emac_print_link_status(dev);
1171         } else
1172                 netif_carrier_on(dev->ndev);
1173
1174         /* Required for Pause packet support in EMAC */
1175         dev_mc_add(ndev, default_mcast_addr, sizeof(default_mcast_addr), 1);
1176
1177         emac_configure(dev);
1178         mal_poll_add(dev->mal, &dev->commac);
1179         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
1180         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(ndev->mtu));
1181         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1182         emac_tx_enable(dev);
1183         emac_rx_enable(dev);
1184         emac_netif_start(dev);
1185
1186         mutex_unlock(&dev->link_lock);
1187
1188         return 0;
1189  oom:
1190         emac_clean_rx_ring(dev);
1191         free_irq(dev->emac_irq, dev);
1192
1193         return -ENOMEM;
1194 }
1195
1196 /* BHs disabled */
1197 #if 0
1198 static int emac_link_differs(struct emac_instance *dev)
1199 {
1200         u32 r = in_be32(&dev->emacp->mr1);
1201
1202         int duplex = r & EMAC_MR1_FDE ? DUPLEX_FULL : DUPLEX_HALF;
1203         int speed, pause, asym_pause;
1204
1205         if (r & EMAC_MR1_MF_1000)
1206                 speed = SPEED_1000;
1207         else if (r & EMAC_MR1_MF_100)
1208                 speed = SPEED_100;
1209         else
1210                 speed = SPEED_10;
1211
1212         switch (r & (EMAC_MR1_EIFC | EMAC_MR1_APP)) {
1213         case (EMAC_MR1_EIFC | EMAC_MR1_APP):
1214                 pause = 1;
1215                 asym_pause = 0;
1216                 break;
1217         case EMAC_MR1_APP:
1218                 pause = 0;
1219                 asym_pause = 1;
1220                 break;
1221         default:
1222                 pause = asym_pause = 0;
1223         }
1224         return speed != dev->phy.speed || duplex != dev->phy.duplex ||
1225             pause != dev->phy.pause || asym_pause != dev->phy.asym_pause;
1226 }
1227 #endif
1228
1229 static void emac_link_timer(struct work_struct *work)
1230 {
1231         struct emac_instance *dev =
1232                 container_of((struct delayed_work *)work,
1233                              struct emac_instance, link_work);
1234         int link_poll_interval;
1235
1236         mutex_lock(&dev->link_lock);
1237         DBG2(dev, "link timer" NL);
1238
1239         if (!dev->opened)
1240                 goto bail;
1241
1242         if (dev->phy.def->ops->poll_link(&dev->phy)) {
1243                 if (!netif_carrier_ok(dev->ndev)) {
1244                         emac_rx_clk_default(dev);
1245                         /* Get new link parameters */
1246                         dev->phy.def->ops->read_link(&dev->phy);
1247
1248                         netif_carrier_on(dev->ndev);
1249                         emac_netif_stop(dev);
1250                         emac_full_tx_reset(dev);
1251                         emac_netif_start(dev);
1252                         emac_print_link_status(dev);
1253                 }
1254                 link_poll_interval = PHY_POLL_LINK_ON;
1255         } else {
1256                 if (netif_carrier_ok(dev->ndev)) {
1257                         emac_rx_clk_tx(dev);
1258                         netif_carrier_off(dev->ndev);
1259                         netif_tx_disable(dev->ndev);
1260                         emac_reinitialize(dev);
1261                         emac_print_link_status(dev);
1262                 }
1263                 link_poll_interval = PHY_POLL_LINK_OFF;
1264         }
1265         schedule_delayed_work(&dev->link_work, link_poll_interval);
1266  bail:
1267         mutex_unlock(&dev->link_lock);
1268 }
1269
1270 static void emac_force_link_update(struct emac_instance *dev)
1271 {
1272         netif_carrier_off(dev->ndev);
1273         smp_rmb();
1274         if (dev->link_polling) {
1275                 cancel_rearming_delayed_work(&dev->link_work);
1276                 if (dev->link_polling)
1277                         schedule_delayed_work(&dev->link_work,  PHY_POLL_LINK_OFF);
1278         }
1279 }
1280
1281 /* Process ctx, rtnl_lock semaphore */
1282 static int emac_close(struct net_device *ndev)
1283 {
1284         struct emac_instance *dev = netdev_priv(ndev);
1285
1286         DBG(dev, "close" NL);
1287
1288         if (dev->phy.address >= 0) {
1289                 dev->link_polling = 0;
1290                 cancel_rearming_delayed_work(&dev->link_work);
1291         }
1292         mutex_lock(&dev->link_lock);
1293         emac_netif_stop(dev);
1294         dev->opened = 0;
1295         mutex_unlock(&dev->link_lock);
1296
1297         emac_rx_disable(dev);
1298         emac_tx_disable(dev);
1299         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
1300         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
1301         mal_poll_del(dev->mal, &dev->commac);
1302
1303         emac_clean_tx_ring(dev);
1304         emac_clean_rx_ring(dev);
1305
1306         free_irq(dev->emac_irq, dev);
1307
1308         return 0;
1309 }
1310
1311 static inline u16 emac_tx_csum(struct emac_instance *dev,
1312                                struct sk_buff *skb)
1313 {
1314         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
1315                 (skb->ip_summed == CHECKSUM_PARTIAL)) {
1316                 ++dev->stats.tx_packets_csum;
1317                 return EMAC_TX_CTRL_TAH_CSUM;
1318         }
1319         return 0;
1320 }
1321
1322 static inline int emac_xmit_finish(struct emac_instance *dev, int len)
1323 {
1324         struct emac_regs __iomem *p = dev->emacp;
1325         struct net_device *ndev = dev->ndev;
1326
1327         /* Send the packet out. If the if makes a significant perf
1328          * difference, then we can store the TMR0 value in "dev"
1329          * instead
1330          */
1331         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
1332                 out_be32(&p->tmr0, EMAC4_TMR0_XMIT);
1333         else
1334                 out_be32(&p->tmr0, EMAC_TMR0_XMIT);
1335
1336         if (unlikely(++dev->tx_cnt == NUM_TX_BUFF)) {
1337                 netif_stop_queue(ndev);
1338                 DBG2(dev, "stopped TX queue" NL);
1339         }
1340
1341         ndev->trans_start = jiffies;
1342         ++dev->stats.tx_packets;
1343         dev->stats.tx_bytes += len;
1344
1345         return 0;
1346 }
1347
1348 /* Tx lock BH */
1349 static int emac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
1350 {
1351         struct emac_instance *dev = netdev_priv(ndev);
1352         unsigned int len = skb->len;
1353         int slot;
1354
1355         u16 ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1356             MAL_TX_CTRL_LAST | emac_tx_csum(dev, skb);
1357
1358         slot = dev->tx_slot++;
1359         if (dev->tx_slot == NUM_TX_BUFF) {
1360                 dev->tx_slot = 0;
1361                 ctrl |= MAL_TX_CTRL_WRAP;
1362         }
1363
1364         DBG2(dev, "xmit(%u) %d" NL, len, slot);
1365
1366         dev->tx_skb[slot] = skb;
1367         dev->tx_desc[slot].data_ptr = dma_map_single(&dev->ofdev->dev,
1368                                                      skb->data, len,
1369                                                      DMA_TO_DEVICE);
1370         dev->tx_desc[slot].data_len = (u16) len;
1371         wmb();
1372         dev->tx_desc[slot].ctrl = ctrl;
1373
1374         return emac_xmit_finish(dev, len);
1375 }
1376
1377 static inline int emac_xmit_split(struct emac_instance *dev, int slot,
1378                                   u32 pd, int len, int last, u16 base_ctrl)
1379 {
1380         while (1) {
1381                 u16 ctrl = base_ctrl;
1382                 int chunk = min(len, MAL_MAX_TX_SIZE);
1383                 len -= chunk;
1384
1385                 slot = (slot + 1) % NUM_TX_BUFF;
1386
1387                 if (last && !len)
1388                         ctrl |= MAL_TX_CTRL_LAST;
1389                 if (slot == NUM_TX_BUFF - 1)
1390                         ctrl |= MAL_TX_CTRL_WRAP;
1391
1392                 dev->tx_skb[slot] = NULL;
1393                 dev->tx_desc[slot].data_ptr = pd;
1394                 dev->tx_desc[slot].data_len = (u16) chunk;
1395                 dev->tx_desc[slot].ctrl = ctrl;
1396                 ++dev->tx_cnt;
1397
1398                 if (!len)
1399                         break;
1400
1401                 pd += chunk;
1402         }
1403         return slot;
1404 }
1405
1406 /* Tx lock BH disabled (SG version for TAH equipped EMACs) */
1407 static int emac_start_xmit_sg(struct sk_buff *skb, struct net_device *ndev)
1408 {
1409         struct emac_instance *dev = netdev_priv(ndev);
1410         int nr_frags = skb_shinfo(skb)->nr_frags;
1411         int len = skb->len, chunk;
1412         int slot, i;
1413         u16 ctrl;
1414         u32 pd;
1415
1416         /* This is common "fast" path */
1417         if (likely(!nr_frags && len <= MAL_MAX_TX_SIZE))
1418                 return emac_start_xmit(skb, ndev);
1419
1420         len -= skb->data_len;
1421
1422         /* Note, this is only an *estimation*, we can still run out of empty
1423          * slots because of the additional fragmentation into
1424          * MAL_MAX_TX_SIZE-sized chunks
1425          */
1426         if (unlikely(dev->tx_cnt + nr_frags + mal_tx_chunks(len) > NUM_TX_BUFF))
1427                 goto stop_queue;
1428
1429         ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1430             emac_tx_csum(dev, skb);
1431         slot = dev->tx_slot;
1432
1433         /* skb data */
1434         dev->tx_skb[slot] = NULL;
1435         chunk = min(len, MAL_MAX_TX_SIZE);
1436         dev->tx_desc[slot].data_ptr = pd =
1437             dma_map_single(&dev->ofdev->dev, skb->data, len, DMA_TO_DEVICE);
1438         dev->tx_desc[slot].data_len = (u16) chunk;
1439         len -= chunk;
1440         if (unlikely(len))
1441                 slot = emac_xmit_split(dev, slot, pd + chunk, len, !nr_frags,
1442                                        ctrl);
1443         /* skb fragments */
1444         for (i = 0; i < nr_frags; ++i) {
1445                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
1446                 len = frag->size;
1447
1448                 if (unlikely(dev->tx_cnt + mal_tx_chunks(len) >= NUM_TX_BUFF))
1449                         goto undo_frame;
1450
1451                 pd = dma_map_page(&dev->ofdev->dev, frag->page, frag->page_offset, len,
1452                                   DMA_TO_DEVICE);
1453
1454                 slot = emac_xmit_split(dev, slot, pd, len, i == nr_frags - 1,
1455                                        ctrl);
1456         }
1457
1458         DBG2(dev, "xmit_sg(%u) %d - %d" NL, skb->len, dev->tx_slot, slot);
1459
1460         /* Attach skb to the last slot so we don't release it too early */
1461         dev->tx_skb[slot] = skb;
1462
1463         /* Send the packet out */
1464         if (dev->tx_slot == NUM_TX_BUFF - 1)
1465                 ctrl |= MAL_TX_CTRL_WRAP;
1466         wmb();
1467         dev->tx_desc[dev->tx_slot].ctrl = ctrl;
1468         dev->tx_slot = (slot + 1) % NUM_TX_BUFF;
1469
1470         return emac_xmit_finish(dev, skb->len);
1471
1472  undo_frame:
1473         /* Well, too bad. Our previous estimation was overly optimistic.
1474          * Undo everything.
1475          */
1476         while (slot != dev->tx_slot) {
1477                 dev->tx_desc[slot].ctrl = 0;
1478                 --dev->tx_cnt;
1479                 if (--slot < 0)
1480                         slot = NUM_TX_BUFF - 1;
1481         }
1482         ++dev->estats.tx_undo;
1483
1484  stop_queue:
1485         netif_stop_queue(ndev);
1486         DBG2(dev, "stopped TX queue" NL);
1487         return 1;
1488 }
1489
1490 /* Tx lock BHs */
1491 static void emac_parse_tx_error(struct emac_instance *dev, u16 ctrl)
1492 {
1493         struct emac_error_stats *st = &dev->estats;
1494
1495         DBG(dev, "BD TX error %04x" NL, ctrl);
1496
1497         ++st->tx_bd_errors;
1498         if (ctrl & EMAC_TX_ST_BFCS)
1499                 ++st->tx_bd_bad_fcs;
1500         if (ctrl & EMAC_TX_ST_LCS)
1501                 ++st->tx_bd_carrier_loss;
1502         if (ctrl & EMAC_TX_ST_ED)
1503                 ++st->tx_bd_excessive_deferral;
1504         if (ctrl & EMAC_TX_ST_EC)
1505                 ++st->tx_bd_excessive_collisions;
1506         if (ctrl & EMAC_TX_ST_LC)
1507                 ++st->tx_bd_late_collision;
1508         if (ctrl & EMAC_TX_ST_MC)
1509                 ++st->tx_bd_multple_collisions;
1510         if (ctrl & EMAC_TX_ST_SC)
1511                 ++st->tx_bd_single_collision;
1512         if (ctrl & EMAC_TX_ST_UR)
1513                 ++st->tx_bd_underrun;
1514         if (ctrl & EMAC_TX_ST_SQE)
1515                 ++st->tx_bd_sqe;
1516 }
1517
1518 static void emac_poll_tx(void *param)
1519 {
1520         struct emac_instance *dev = param;
1521         u32 bad_mask;
1522
1523         DBG2(dev, "poll_tx, %d %d" NL, dev->tx_cnt, dev->ack_slot);
1524
1525         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
1526                 bad_mask = EMAC_IS_BAD_TX_TAH;
1527         else
1528                 bad_mask = EMAC_IS_BAD_TX;
1529
1530         netif_tx_lock_bh(dev->ndev);
1531         if (dev->tx_cnt) {
1532                 u16 ctrl;
1533                 int slot = dev->ack_slot, n = 0;
1534         again:
1535                 ctrl = dev->tx_desc[slot].ctrl;
1536                 if (!(ctrl & MAL_TX_CTRL_READY)) {
1537                         struct sk_buff *skb = dev->tx_skb[slot];
1538                         ++n;
1539
1540                         if (skb) {
1541                                 dev_kfree_skb(skb);
1542                                 dev->tx_skb[slot] = NULL;
1543                         }
1544                         slot = (slot + 1) % NUM_TX_BUFF;
1545
1546                         if (unlikely(ctrl & bad_mask))
1547                                 emac_parse_tx_error(dev, ctrl);
1548
1549                         if (--dev->tx_cnt)
1550                                 goto again;
1551                 }
1552                 if (n) {
1553                         dev->ack_slot = slot;
1554                         if (netif_queue_stopped(dev->ndev) &&
1555                             dev->tx_cnt < EMAC_TX_WAKEUP_THRESH)
1556                                 netif_wake_queue(dev->ndev);
1557
1558                         DBG2(dev, "tx %d pkts" NL, n);
1559                 }
1560         }
1561         netif_tx_unlock_bh(dev->ndev);
1562 }
1563
1564 static inline void emac_recycle_rx_skb(struct emac_instance *dev, int slot,
1565                                        int len)
1566 {
1567         struct sk_buff *skb = dev->rx_skb[slot];
1568
1569         DBG2(dev, "recycle %d %d" NL, slot, len);
1570
1571         if (len)
1572                 dma_map_single(&dev->ofdev->dev, skb->data - 2,
1573                                EMAC_DMA_ALIGN(len + 2), DMA_FROM_DEVICE);
1574
1575         dev->rx_desc[slot].data_len = 0;
1576         wmb();
1577         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1578             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1579 }
1580
1581 static void emac_parse_rx_error(struct emac_instance *dev, u16 ctrl)
1582 {
1583         struct emac_error_stats *st = &dev->estats;
1584
1585         DBG(dev, "BD RX error %04x" NL, ctrl);
1586
1587         ++st->rx_bd_errors;
1588         if (ctrl & EMAC_RX_ST_OE)
1589                 ++st->rx_bd_overrun;
1590         if (ctrl & EMAC_RX_ST_BP)
1591                 ++st->rx_bd_bad_packet;
1592         if (ctrl & EMAC_RX_ST_RP)
1593                 ++st->rx_bd_runt_packet;
1594         if (ctrl & EMAC_RX_ST_SE)
1595                 ++st->rx_bd_short_event;
1596         if (ctrl & EMAC_RX_ST_AE)
1597                 ++st->rx_bd_alignment_error;
1598         if (ctrl & EMAC_RX_ST_BFCS)
1599                 ++st->rx_bd_bad_fcs;
1600         if (ctrl & EMAC_RX_ST_PTL)
1601                 ++st->rx_bd_packet_too_long;
1602         if (ctrl & EMAC_RX_ST_ORE)
1603                 ++st->rx_bd_out_of_range;
1604         if (ctrl & EMAC_RX_ST_IRE)
1605                 ++st->rx_bd_in_range;
1606 }
1607
1608 static inline void emac_rx_csum(struct emac_instance *dev,
1609                                 struct sk_buff *skb, u16 ctrl)
1610 {
1611 #ifdef CONFIG_IBM_NEW_EMAC_TAH
1612         if (!ctrl && dev->tah_dev) {
1613                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1614                 ++dev->stats.rx_packets_csum;
1615         }
1616 #endif
1617 }
1618
1619 static inline int emac_rx_sg_append(struct emac_instance *dev, int slot)
1620 {
1621         if (likely(dev->rx_sg_skb != NULL)) {
1622                 int len = dev->rx_desc[slot].data_len;
1623                 int tot_len = dev->rx_sg_skb->len + len;
1624
1625                 if (unlikely(tot_len + 2 > dev->rx_skb_size)) {
1626                         ++dev->estats.rx_dropped_mtu;
1627                         dev_kfree_skb(dev->rx_sg_skb);
1628                         dev->rx_sg_skb = NULL;
1629                 } else {
1630                         cacheable_memcpy(skb_tail_pointer(dev->rx_sg_skb),
1631                                          dev->rx_skb[slot]->data, len);
1632                         skb_put(dev->rx_sg_skb, len);
1633                         emac_recycle_rx_skb(dev, slot, len);
1634                         return 0;
1635                 }
1636         }
1637         emac_recycle_rx_skb(dev, slot, 0);
1638         return -1;
1639 }
1640
1641 /* NAPI poll context */
1642 static int emac_poll_rx(void *param, int budget)
1643 {
1644         struct emac_instance *dev = param;
1645         int slot = dev->rx_slot, received = 0;
1646
1647         DBG2(dev, "poll_rx(%d)" NL, budget);
1648
1649  again:
1650         while (budget > 0) {
1651                 int len;
1652                 struct sk_buff *skb;
1653                 u16 ctrl = dev->rx_desc[slot].ctrl;
1654
1655                 if (ctrl & MAL_RX_CTRL_EMPTY)
1656                         break;
1657
1658                 skb = dev->rx_skb[slot];
1659                 mb();
1660                 len = dev->rx_desc[slot].data_len;
1661
1662                 if (unlikely(!MAL_IS_SINGLE_RX(ctrl)))
1663                         goto sg;
1664
1665                 ctrl &= EMAC_BAD_RX_MASK;
1666                 if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1667                         emac_parse_rx_error(dev, ctrl);
1668                         ++dev->estats.rx_dropped_error;
1669                         emac_recycle_rx_skb(dev, slot, 0);
1670                         len = 0;
1671                         goto next;
1672                 }
1673
1674                 if (len < ETH_HLEN) {
1675                         ++dev->estats.rx_dropped_stack;
1676                         emac_recycle_rx_skb(dev, slot, len);
1677                         goto next;
1678                 }
1679
1680                 if (len && len < EMAC_RX_COPY_THRESH) {
1681                         struct sk_buff *copy_skb =
1682                             alloc_skb(len + EMAC_RX_SKB_HEADROOM + 2, GFP_ATOMIC);
1683                         if (unlikely(!copy_skb))
1684                                 goto oom;
1685
1686                         skb_reserve(copy_skb, EMAC_RX_SKB_HEADROOM + 2);
1687                         cacheable_memcpy(copy_skb->data - 2, skb->data - 2,
1688                                          len + 2);
1689                         emac_recycle_rx_skb(dev, slot, len);
1690                         skb = copy_skb;
1691                 } else if (unlikely(emac_alloc_rx_skb(dev, slot, GFP_ATOMIC)))
1692                         goto oom;
1693
1694                 skb_put(skb, len);
1695         push_packet:
1696                 skb->dev = dev->ndev;
1697                 skb->protocol = eth_type_trans(skb, dev->ndev);
1698                 emac_rx_csum(dev, skb, ctrl);
1699
1700                 if (unlikely(netif_receive_skb(skb) == NET_RX_DROP))
1701                         ++dev->estats.rx_dropped_stack;
1702         next:
1703                 ++dev->stats.rx_packets;
1704         skip:
1705                 dev->stats.rx_bytes += len;
1706                 slot = (slot + 1) % NUM_RX_BUFF;
1707                 --budget;
1708                 ++received;
1709                 continue;
1710         sg:
1711                 if (ctrl & MAL_RX_CTRL_FIRST) {
1712                         BUG_ON(dev->rx_sg_skb);
1713                         if (unlikely(emac_alloc_rx_skb(dev, slot, GFP_ATOMIC))) {
1714                                 DBG(dev, "rx OOM %d" NL, slot);
1715                                 ++dev->estats.rx_dropped_oom;
1716                                 emac_recycle_rx_skb(dev, slot, 0);
1717                         } else {
1718                                 dev->rx_sg_skb = skb;
1719                                 skb_put(skb, len);
1720                         }
1721                 } else if (!emac_rx_sg_append(dev, slot) &&
1722                            (ctrl & MAL_RX_CTRL_LAST)) {
1723
1724                         skb = dev->rx_sg_skb;
1725                         dev->rx_sg_skb = NULL;
1726
1727                         ctrl &= EMAC_BAD_RX_MASK;
1728                         if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1729                                 emac_parse_rx_error(dev, ctrl);
1730                                 ++dev->estats.rx_dropped_error;
1731                                 dev_kfree_skb(skb);
1732                                 len = 0;
1733                         } else
1734                                 goto push_packet;
1735                 }
1736                 goto skip;
1737         oom:
1738                 DBG(dev, "rx OOM %d" NL, slot);
1739                 /* Drop the packet and recycle skb */
1740                 ++dev->estats.rx_dropped_oom;
1741                 emac_recycle_rx_skb(dev, slot, 0);
1742                 goto next;
1743         }
1744
1745         if (received) {
1746                 DBG2(dev, "rx %d BDs" NL, received);
1747                 dev->rx_slot = slot;
1748         }
1749
1750         if (unlikely(budget && test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags))) {
1751                 mb();
1752                 if (!(dev->rx_desc[slot].ctrl & MAL_RX_CTRL_EMPTY)) {
1753                         DBG2(dev, "rx restart" NL);
1754                         received = 0;
1755                         goto again;
1756                 }
1757
1758                 if (dev->rx_sg_skb) {
1759                         DBG2(dev, "dropping partial rx packet" NL);
1760                         ++dev->estats.rx_dropped_error;
1761                         dev_kfree_skb(dev->rx_sg_skb);
1762                         dev->rx_sg_skb = NULL;
1763                 }
1764
1765                 clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1766                 mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1767                 emac_rx_enable(dev);
1768                 dev->rx_slot = 0;
1769         }
1770         return received;
1771 }
1772
1773 /* NAPI poll context */
1774 static int emac_peek_rx(void *param)
1775 {
1776         struct emac_instance *dev = param;
1777
1778         return !(dev->rx_desc[dev->rx_slot].ctrl & MAL_RX_CTRL_EMPTY);
1779 }
1780
1781 /* NAPI poll context */
1782 static int emac_peek_rx_sg(void *param)
1783 {
1784         struct emac_instance *dev = param;
1785
1786         int slot = dev->rx_slot;
1787         while (1) {
1788                 u16 ctrl = dev->rx_desc[slot].ctrl;
1789                 if (ctrl & MAL_RX_CTRL_EMPTY)
1790                         return 0;
1791                 else if (ctrl & MAL_RX_CTRL_LAST)
1792                         return 1;
1793
1794                 slot = (slot + 1) % NUM_RX_BUFF;
1795
1796                 /* I'm just being paranoid here :) */
1797                 if (unlikely(slot == dev->rx_slot))
1798                         return 0;
1799         }
1800 }
1801
1802 /* Hard IRQ */
1803 static void emac_rxde(void *param)
1804 {
1805         struct emac_instance *dev = param;
1806
1807         ++dev->estats.rx_stopped;
1808         emac_rx_disable_async(dev);
1809 }
1810
1811 /* Hard IRQ */
1812 static irqreturn_t emac_irq(int irq, void *dev_instance)
1813 {
1814         struct emac_instance *dev = dev_instance;
1815         struct emac_regs __iomem *p = dev->emacp;
1816         struct emac_error_stats *st = &dev->estats;
1817         u32 isr;
1818
1819         spin_lock(&dev->lock);
1820
1821         isr = in_be32(&p->isr);
1822         out_be32(&p->isr, isr);
1823
1824         DBG(dev, "isr = %08x" NL, isr);
1825
1826         if (isr & EMAC4_ISR_TXPE)
1827                 ++st->tx_parity;
1828         if (isr & EMAC4_ISR_RXPE)
1829                 ++st->rx_parity;
1830         if (isr & EMAC4_ISR_TXUE)
1831                 ++st->tx_underrun;
1832         if (isr & EMAC4_ISR_RXOE)
1833                 ++st->rx_fifo_overrun;
1834         if (isr & EMAC_ISR_OVR)
1835                 ++st->rx_overrun;
1836         if (isr & EMAC_ISR_BP)
1837                 ++st->rx_bad_packet;
1838         if (isr & EMAC_ISR_RP)
1839                 ++st->rx_runt_packet;
1840         if (isr & EMAC_ISR_SE)
1841                 ++st->rx_short_event;
1842         if (isr & EMAC_ISR_ALE)
1843                 ++st->rx_alignment_error;
1844         if (isr & EMAC_ISR_BFCS)
1845                 ++st->rx_bad_fcs;
1846         if (isr & EMAC_ISR_PTLE)
1847                 ++st->rx_packet_too_long;
1848         if (isr & EMAC_ISR_ORE)
1849                 ++st->rx_out_of_range;
1850         if (isr & EMAC_ISR_IRE)
1851                 ++st->rx_in_range;
1852         if (isr & EMAC_ISR_SQE)
1853                 ++st->tx_sqe;
1854         if (isr & EMAC_ISR_TE)
1855                 ++st->tx_errors;
1856
1857         spin_unlock(&dev->lock);
1858
1859         return IRQ_HANDLED;
1860 }
1861
1862 static struct net_device_stats *emac_stats(struct net_device *ndev)
1863 {
1864         struct emac_instance *dev = netdev_priv(ndev);
1865         struct emac_stats *st = &dev->stats;
1866         struct emac_error_stats *est = &dev->estats;
1867         struct net_device_stats *nst = &dev->nstats;
1868         unsigned long flags;
1869
1870         DBG2(dev, "stats" NL);
1871
1872         /* Compute "legacy" statistics */
1873         spin_lock_irqsave(&dev->lock, flags);
1874         nst->rx_packets = (unsigned long)st->rx_packets;
1875         nst->rx_bytes = (unsigned long)st->rx_bytes;
1876         nst->tx_packets = (unsigned long)st->tx_packets;
1877         nst->tx_bytes = (unsigned long)st->tx_bytes;
1878         nst->rx_dropped = (unsigned long)(est->rx_dropped_oom +
1879                                           est->rx_dropped_error +
1880                                           est->rx_dropped_resize +
1881                                           est->rx_dropped_mtu);
1882         nst->tx_dropped = (unsigned long)est->tx_dropped;
1883
1884         nst->rx_errors = (unsigned long)est->rx_bd_errors;
1885         nst->rx_fifo_errors = (unsigned long)(est->rx_bd_overrun +
1886                                               est->rx_fifo_overrun +
1887                                               est->rx_overrun);
1888         nst->rx_frame_errors = (unsigned long)(est->rx_bd_alignment_error +
1889                                                est->rx_alignment_error);
1890         nst->rx_crc_errors = (unsigned long)(est->rx_bd_bad_fcs +
1891                                              est->rx_bad_fcs);
1892         nst->rx_length_errors = (unsigned long)(est->rx_bd_runt_packet +
1893                                                 est->rx_bd_short_event +
1894                                                 est->rx_bd_packet_too_long +
1895                                                 est->rx_bd_out_of_range +
1896                                                 est->rx_bd_in_range +
1897                                                 est->rx_runt_packet +
1898                                                 est->rx_short_event +
1899                                                 est->rx_packet_too_long +
1900                                                 est->rx_out_of_range +
1901                                                 est->rx_in_range);
1902
1903         nst->tx_errors = (unsigned long)(est->tx_bd_errors + est->tx_errors);
1904         nst->tx_fifo_errors = (unsigned long)(est->tx_bd_underrun +
1905                                               est->tx_underrun);
1906         nst->tx_carrier_errors = (unsigned long)est->tx_bd_carrier_loss;
1907         nst->collisions = (unsigned long)(est->tx_bd_excessive_deferral +
1908                                           est->tx_bd_excessive_collisions +
1909                                           est->tx_bd_late_collision +
1910                                           est->tx_bd_multple_collisions);
1911         spin_unlock_irqrestore(&dev->lock, flags);
1912         return nst;
1913 }
1914
1915 static struct mal_commac_ops emac_commac_ops = {
1916         .poll_tx = &emac_poll_tx,
1917         .poll_rx = &emac_poll_rx,
1918         .peek_rx = &emac_peek_rx,
1919         .rxde = &emac_rxde,
1920 };
1921
1922 static struct mal_commac_ops emac_commac_sg_ops = {
1923         .poll_tx = &emac_poll_tx,
1924         .poll_rx = &emac_poll_rx,
1925         .peek_rx = &emac_peek_rx_sg,
1926         .rxde = &emac_rxde,
1927 };
1928
1929 /* Ethtool support */
1930 static int emac_ethtool_get_settings(struct net_device *ndev,
1931                                      struct ethtool_cmd *cmd)
1932 {
1933         struct emac_instance *dev = netdev_priv(ndev);
1934
1935         cmd->supported = dev->phy.features;
1936         cmd->port = PORT_MII;
1937         cmd->phy_address = dev->phy.address;
1938         cmd->transceiver =
1939             dev->phy.address >= 0 ? XCVR_EXTERNAL : XCVR_INTERNAL;
1940
1941         mutex_lock(&dev->link_lock);
1942         cmd->advertising = dev->phy.advertising;
1943         cmd->autoneg = dev->phy.autoneg;
1944         cmd->speed = dev->phy.speed;
1945         cmd->duplex = dev->phy.duplex;
1946         mutex_unlock(&dev->link_lock);
1947
1948         return 0;
1949 }
1950
1951 static int emac_ethtool_set_settings(struct net_device *ndev,
1952                                      struct ethtool_cmd *cmd)
1953 {
1954         struct emac_instance *dev = netdev_priv(ndev);
1955         u32 f = dev->phy.features;
1956
1957         DBG(dev, "set_settings(%d, %d, %d, 0x%08x)" NL,
1958             cmd->autoneg, cmd->speed, cmd->duplex, cmd->advertising);
1959
1960         /* Basic sanity checks */
1961         if (dev->phy.address < 0)
1962                 return -EOPNOTSUPP;
1963         if (cmd->autoneg != AUTONEG_ENABLE && cmd->autoneg != AUTONEG_DISABLE)
1964                 return -EINVAL;
1965         if (cmd->autoneg == AUTONEG_ENABLE && cmd->advertising == 0)
1966                 return -EINVAL;
1967         if (cmd->duplex != DUPLEX_HALF && cmd->duplex != DUPLEX_FULL)
1968                 return -EINVAL;
1969
1970         if (cmd->autoneg == AUTONEG_DISABLE) {
1971                 switch (cmd->speed) {
1972                 case SPEED_10:
1973                         if (cmd->duplex == DUPLEX_HALF
1974                             && !(f & SUPPORTED_10baseT_Half))
1975                                 return -EINVAL;
1976                         if (cmd->duplex == DUPLEX_FULL
1977                             && !(f & SUPPORTED_10baseT_Full))
1978                                 return -EINVAL;
1979                         break;
1980                 case SPEED_100:
1981                         if (cmd->duplex == DUPLEX_HALF
1982                             && !(f & SUPPORTED_100baseT_Half))
1983                                 return -EINVAL;
1984                         if (cmd->duplex == DUPLEX_FULL
1985                             && !(f & SUPPORTED_100baseT_Full))
1986                                 return -EINVAL;
1987                         break;
1988                 case SPEED_1000:
1989                         if (cmd->duplex == DUPLEX_HALF
1990                             && !(f & SUPPORTED_1000baseT_Half))
1991                                 return -EINVAL;
1992                         if (cmd->duplex == DUPLEX_FULL
1993                             && !(f & SUPPORTED_1000baseT_Full))
1994                                 return -EINVAL;
1995                         break;
1996                 default:
1997                         return -EINVAL;
1998                 }
1999
2000                 mutex_lock(&dev->link_lock);
2001                 dev->phy.def->ops->setup_forced(&dev->phy, cmd->speed,
2002                                                 cmd->duplex);
2003                 mutex_unlock(&dev->link_lock);
2004
2005         } else {
2006                 if (!(f & SUPPORTED_Autoneg))
2007                         return -EINVAL;
2008
2009                 mutex_lock(&dev->link_lock);
2010                 dev->phy.def->ops->setup_aneg(&dev->phy,
2011                                               (cmd->advertising & f) |
2012                                               (dev->phy.advertising &
2013                                                (ADVERTISED_Pause |
2014                                                 ADVERTISED_Asym_Pause)));
2015                 mutex_unlock(&dev->link_lock);
2016         }
2017         emac_force_link_update(dev);
2018
2019         return 0;
2020 }
2021
2022 static void emac_ethtool_get_ringparam(struct net_device *ndev,
2023                                        struct ethtool_ringparam *rp)
2024 {
2025         rp->rx_max_pending = rp->rx_pending = NUM_RX_BUFF;
2026         rp->tx_max_pending = rp->tx_pending = NUM_TX_BUFF;
2027 }
2028
2029 static void emac_ethtool_get_pauseparam(struct net_device *ndev,
2030                                         struct ethtool_pauseparam *pp)
2031 {
2032         struct emac_instance *dev = netdev_priv(ndev);
2033
2034         mutex_lock(&dev->link_lock);
2035         if ((dev->phy.features & SUPPORTED_Autoneg) &&
2036             (dev->phy.advertising & (ADVERTISED_Pause | ADVERTISED_Asym_Pause)))
2037                 pp->autoneg = 1;
2038
2039         if (dev->phy.duplex == DUPLEX_FULL) {
2040                 if (dev->phy.pause)
2041                         pp->rx_pause = pp->tx_pause = 1;
2042                 else if (dev->phy.asym_pause)
2043                         pp->tx_pause = 1;
2044         }
2045         mutex_unlock(&dev->link_lock);
2046 }
2047
2048 static u32 emac_ethtool_get_rx_csum(struct net_device *ndev)
2049 {
2050         struct emac_instance *dev = netdev_priv(ndev);
2051
2052         return dev->tah_dev != NULL;
2053 }
2054
2055 static int emac_get_regs_len(struct emac_instance *dev)
2056 {
2057         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
2058                 return sizeof(struct emac_ethtool_regs_subhdr) +
2059                         EMAC4_ETHTOOL_REGS_SIZE(dev);
2060         else
2061                 return sizeof(struct emac_ethtool_regs_subhdr) +
2062                         EMAC_ETHTOOL_REGS_SIZE(dev);
2063 }
2064
2065 static int emac_ethtool_get_regs_len(struct net_device *ndev)
2066 {
2067         struct emac_instance *dev = netdev_priv(ndev);
2068         int size;
2069
2070         size = sizeof(struct emac_ethtool_regs_hdr) +
2071                 emac_get_regs_len(dev) + mal_get_regs_len(dev->mal);
2072         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2073                 size += zmii_get_regs_len(dev->zmii_dev);
2074         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2075                 size += rgmii_get_regs_len(dev->rgmii_dev);
2076         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2077                 size += tah_get_regs_len(dev->tah_dev);
2078
2079         return size;
2080 }
2081
2082 static void *emac_dump_regs(struct emac_instance *dev, void *buf)
2083 {
2084         struct emac_ethtool_regs_subhdr *hdr = buf;
2085
2086         hdr->index = dev->cell_index;
2087         if (emac_has_feature(dev, EMAC_FTR_EMAC4)) {
2088                 hdr->version = EMAC4_ETHTOOL_REGS_VER;
2089                 memcpy_fromio(hdr + 1, dev->emacp, EMAC4_ETHTOOL_REGS_SIZE(dev));
2090                 return ((void *)(hdr + 1) + EMAC4_ETHTOOL_REGS_SIZE(dev));
2091         } else {
2092                 hdr->version = EMAC_ETHTOOL_REGS_VER;
2093                 memcpy_fromio(hdr + 1, dev->emacp, EMAC_ETHTOOL_REGS_SIZE(dev));
2094                 return ((void *)(hdr + 1) + EMAC_ETHTOOL_REGS_SIZE(dev));
2095         }
2096 }
2097
2098 static void emac_ethtool_get_regs(struct net_device *ndev,
2099                                   struct ethtool_regs *regs, void *buf)
2100 {
2101         struct emac_instance *dev = netdev_priv(ndev);
2102         struct emac_ethtool_regs_hdr *hdr = buf;
2103
2104         hdr->components = 0;
2105         buf = hdr + 1;
2106
2107         buf = mal_dump_regs(dev->mal, buf);
2108         buf = emac_dump_regs(dev, buf);
2109         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII)) {
2110                 hdr->components |= EMAC_ETHTOOL_REGS_ZMII;
2111                 buf = zmii_dump_regs(dev->zmii_dev, buf);
2112         }
2113         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII)) {
2114                 hdr->components |= EMAC_ETHTOOL_REGS_RGMII;
2115                 buf = rgmii_dump_regs(dev->rgmii_dev, buf);
2116         }
2117         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH)) {
2118                 hdr->components |= EMAC_ETHTOOL_REGS_TAH;
2119                 buf = tah_dump_regs(dev->tah_dev, buf);
2120         }
2121 }
2122
2123 static int emac_ethtool_nway_reset(struct net_device *ndev)
2124 {
2125         struct emac_instance *dev = netdev_priv(ndev);
2126         int res = 0;
2127
2128         DBG(dev, "nway_reset" NL);
2129
2130         if (dev->phy.address < 0)
2131                 return -EOPNOTSUPP;
2132
2133         mutex_lock(&dev->link_lock);
2134         if (!dev->phy.autoneg) {
2135                 res = -EINVAL;
2136                 goto out;
2137         }
2138
2139         dev->phy.def->ops->setup_aneg(&dev->phy, dev->phy.advertising);
2140  out:
2141         mutex_unlock(&dev->link_lock);
2142         emac_force_link_update(dev);
2143         return res;
2144 }
2145
2146 static int emac_ethtool_get_stats_count(struct net_device *ndev)
2147 {
2148         return EMAC_ETHTOOL_STATS_COUNT;
2149 }
2150
2151 static void emac_ethtool_get_strings(struct net_device *ndev, u32 stringset,
2152                                      u8 * buf)
2153 {
2154         if (stringset == ETH_SS_STATS)
2155                 memcpy(buf, &emac_stats_keys, sizeof(emac_stats_keys));
2156 }
2157
2158 static void emac_ethtool_get_ethtool_stats(struct net_device *ndev,
2159                                            struct ethtool_stats *estats,
2160                                            u64 * tmp_stats)
2161 {
2162         struct emac_instance *dev = netdev_priv(ndev);
2163
2164         memcpy(tmp_stats, &dev->stats, sizeof(dev->stats));
2165         tmp_stats += sizeof(dev->stats) / sizeof(u64);
2166         memcpy(tmp_stats, &dev->estats, sizeof(dev->estats));
2167 }
2168
2169 static void emac_ethtool_get_drvinfo(struct net_device *ndev,
2170                                      struct ethtool_drvinfo *info)
2171 {
2172         struct emac_instance *dev = netdev_priv(ndev);
2173
2174         strcpy(info->driver, "ibm_emac");
2175         strcpy(info->version, DRV_VERSION);
2176         info->fw_version[0] = '\0';
2177         sprintf(info->bus_info, "PPC 4xx EMAC-%d %s",
2178                 dev->cell_index, dev->ofdev->node->full_name);
2179         info->n_stats = emac_ethtool_get_stats_count(ndev);
2180         info->regdump_len = emac_ethtool_get_regs_len(ndev);
2181 }
2182
2183 static const struct ethtool_ops emac_ethtool_ops = {
2184         .get_settings = emac_ethtool_get_settings,
2185         .set_settings = emac_ethtool_set_settings,
2186         .get_drvinfo = emac_ethtool_get_drvinfo,
2187
2188         .get_regs_len = emac_ethtool_get_regs_len,
2189         .get_regs = emac_ethtool_get_regs,
2190
2191         .nway_reset = emac_ethtool_nway_reset,
2192
2193         .get_ringparam = emac_ethtool_get_ringparam,
2194         .get_pauseparam = emac_ethtool_get_pauseparam,
2195
2196         .get_rx_csum = emac_ethtool_get_rx_csum,
2197
2198         .get_strings = emac_ethtool_get_strings,
2199         .get_stats_count = emac_ethtool_get_stats_count,
2200         .get_ethtool_stats = emac_ethtool_get_ethtool_stats,
2201
2202         .get_link = ethtool_op_get_link,
2203         .get_tx_csum = ethtool_op_get_tx_csum,
2204         .get_sg = ethtool_op_get_sg,
2205 };
2206
2207 static int emac_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2208 {
2209         struct emac_instance *dev = netdev_priv(ndev);
2210         uint16_t *data = (uint16_t *) & rq->ifr_ifru;
2211
2212         DBG(dev, "ioctl %08x" NL, cmd);
2213
2214         if (dev->phy.address < 0)
2215                 return -EOPNOTSUPP;
2216
2217         switch (cmd) {
2218         case SIOCGMIIPHY:
2219         case SIOCDEVPRIVATE:
2220                 data[0] = dev->phy.address;
2221                 /* Fall through */
2222         case SIOCGMIIREG:
2223         case SIOCDEVPRIVATE + 1:
2224                 data[3] = emac_mdio_read(ndev, dev->phy.address, data[1]);
2225                 return 0;
2226
2227         case SIOCSMIIREG:
2228         case SIOCDEVPRIVATE + 2:
2229                 if (!capable(CAP_NET_ADMIN))
2230                         return -EPERM;
2231                 emac_mdio_write(ndev, dev->phy.address, data[1], data[2]);
2232                 return 0;
2233         default:
2234                 return -EOPNOTSUPP;
2235         }
2236 }
2237
2238 struct emac_depentry {
2239         u32                     phandle;
2240         struct device_node      *node;
2241         struct of_device        *ofdev;
2242         void                    *drvdata;
2243 };
2244
2245 #define EMAC_DEP_MAL_IDX        0
2246 #define EMAC_DEP_ZMII_IDX       1
2247 #define EMAC_DEP_RGMII_IDX      2
2248 #define EMAC_DEP_TAH_IDX        3
2249 #define EMAC_DEP_MDIO_IDX       4
2250 #define EMAC_DEP_PREV_IDX       5
2251 #define EMAC_DEP_COUNT          6
2252
2253 static int __devinit emac_check_deps(struct emac_instance *dev,
2254                                      struct emac_depentry *deps)
2255 {
2256         int i, there = 0;
2257         struct device_node *np;
2258
2259         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2260                 /* no dependency on that item, allright */
2261                 if (deps[i].phandle == 0) {
2262                         there++;
2263                         continue;
2264                 }
2265                 /* special case for blist as the dependency might go away */
2266                 if (i == EMAC_DEP_PREV_IDX) {
2267                         np = *(dev->blist - 1);
2268                         if (np == NULL) {
2269                                 deps[i].phandle = 0;
2270                                 there++;
2271                                 continue;
2272                         }
2273                         if (deps[i].node == NULL)
2274                                 deps[i].node = of_node_get(np);
2275                 }
2276                 if (deps[i].node == NULL)
2277                         deps[i].node = of_find_node_by_phandle(deps[i].phandle);
2278                 if (deps[i].node == NULL)
2279                         continue;
2280                 if (deps[i].ofdev == NULL)
2281                         deps[i].ofdev = of_find_device_by_node(deps[i].node);
2282                 if (deps[i].ofdev == NULL)
2283                         continue;
2284                 if (deps[i].drvdata == NULL)
2285                         deps[i].drvdata = dev_get_drvdata(&deps[i].ofdev->dev);
2286                 if (deps[i].drvdata != NULL)
2287                         there++;
2288         }
2289         return (there == EMAC_DEP_COUNT);
2290 }
2291
2292 static void emac_put_deps(struct emac_instance *dev)
2293 {
2294         if (dev->mal_dev)
2295                 of_dev_put(dev->mal_dev);
2296         if (dev->zmii_dev)
2297                 of_dev_put(dev->zmii_dev);
2298         if (dev->rgmii_dev)
2299                 of_dev_put(dev->rgmii_dev);
2300         if (dev->mdio_dev)
2301                 of_dev_put(dev->mdio_dev);
2302         if (dev->tah_dev)
2303                 of_dev_put(dev->tah_dev);
2304 }
2305
2306 static int __devinit emac_of_bus_notify(struct notifier_block *nb,
2307                                         unsigned long action, void *data)
2308 {
2309         /* We are only intereted in device addition */
2310         if (action == BUS_NOTIFY_BOUND_DRIVER)
2311                 wake_up_all(&emac_probe_wait);
2312         return 0;
2313 }
2314
2315 static struct notifier_block emac_of_bus_notifier __devinitdata = {
2316         .notifier_call = emac_of_bus_notify
2317 };
2318
2319 static int __devinit emac_wait_deps(struct emac_instance *dev)
2320 {
2321         struct emac_depentry deps[EMAC_DEP_COUNT];
2322         int i, err;
2323
2324         memset(&deps, 0, sizeof(deps));
2325
2326         deps[EMAC_DEP_MAL_IDX].phandle = dev->mal_ph;
2327         deps[EMAC_DEP_ZMII_IDX].phandle = dev->zmii_ph;
2328         deps[EMAC_DEP_RGMII_IDX].phandle = dev->rgmii_ph;
2329         if (dev->tah_ph)
2330                 deps[EMAC_DEP_TAH_IDX].phandle = dev->tah_ph;
2331         if (dev->mdio_ph)
2332                 deps[EMAC_DEP_MDIO_IDX].phandle = dev->mdio_ph;
2333         if (dev->blist && dev->blist > emac_boot_list)
2334                 deps[EMAC_DEP_PREV_IDX].phandle = 0xffffffffu;
2335         bus_register_notifier(&of_platform_bus_type, &emac_of_bus_notifier);
2336         wait_event_timeout(emac_probe_wait,
2337                            emac_check_deps(dev, deps),
2338                            EMAC_PROBE_DEP_TIMEOUT);
2339         bus_unregister_notifier(&of_platform_bus_type, &emac_of_bus_notifier);
2340         err = emac_check_deps(dev, deps) ? 0 : -ENODEV;
2341         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2342                 if (deps[i].node)
2343                         of_node_put(deps[i].node);
2344                 if (err && deps[i].ofdev)
2345                         of_dev_put(deps[i].ofdev);
2346         }
2347         if (err == 0) {
2348                 dev->mal_dev = deps[EMAC_DEP_MAL_IDX].ofdev;
2349                 dev->zmii_dev = deps[EMAC_DEP_ZMII_IDX].ofdev;
2350                 dev->rgmii_dev = deps[EMAC_DEP_RGMII_IDX].ofdev;
2351                 dev->tah_dev = deps[EMAC_DEP_TAH_IDX].ofdev;
2352                 dev->mdio_dev = deps[EMAC_DEP_MDIO_IDX].ofdev;
2353         }
2354         if (deps[EMAC_DEP_PREV_IDX].ofdev)
2355                 of_dev_put(deps[EMAC_DEP_PREV_IDX].ofdev);
2356         return err;
2357 }
2358
2359 static int __devinit emac_read_uint_prop(struct device_node *np, const char *name,
2360                                          u32 *val, int fatal)
2361 {
2362         int len;
2363         const u32 *prop = of_get_property(np, name, &len);
2364         if (prop == NULL || len < sizeof(u32)) {
2365                 if (fatal)
2366                         printk(KERN_ERR "%s: missing %s property\n",
2367                                np->full_name, name);
2368                 return -ENODEV;
2369         }
2370         *val = *prop;
2371         return 0;
2372 }
2373
2374 static int __devinit emac_init_phy(struct emac_instance *dev)
2375 {
2376         struct device_node *np = dev->ofdev->node;
2377         struct net_device *ndev = dev->ndev;
2378         u32 phy_map, adv;
2379         int i;
2380
2381         dev->phy.dev = ndev;
2382         dev->phy.mode = dev->phy_mode;
2383
2384         /* PHY-less configuration.
2385          * XXX I probably should move these settings to the dev tree
2386          */
2387         if (dev->phy_address == 0xffffffff && dev->phy_map == 0xffffffff) {
2388                 emac_reset(dev);
2389
2390                 /* PHY-less configuration.
2391                  * XXX I probably should move these settings to the dev tree
2392                  */
2393                 dev->phy.address = -1;
2394                 dev->phy.features = SUPPORTED_MII;
2395                 if (emac_phy_supports_gige(dev->phy_mode))
2396                         dev->phy.features |= SUPPORTED_1000baseT_Full;
2397                 else
2398                         dev->phy.features |= SUPPORTED_100baseT_Full;
2399                 dev->phy.pause = 1;
2400
2401                 return 0;
2402         }
2403
2404         mutex_lock(&emac_phy_map_lock);
2405         phy_map = dev->phy_map | busy_phy_map;
2406
2407         DBG(dev, "PHY maps %08x %08x" NL, dev->phy_map, busy_phy_map);
2408
2409         dev->phy.mdio_read = emac_mdio_read;
2410         dev->phy.mdio_write = emac_mdio_write;
2411
2412         /* Enable internal clock source */
2413 #ifdef CONFIG_PPC_DCR_NATIVE
2414         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2415                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2416 #endif
2417         /* PHY clock workaround */
2418         emac_rx_clk_tx(dev);
2419
2420         /* Enable internal clock source on 440GX*/
2421 #ifdef CONFIG_PPC_DCR_NATIVE
2422         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2423                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2424 #endif
2425         /* Configure EMAC with defaults so we can at least use MDIO
2426          * This is needed mostly for 440GX
2427          */
2428         if (emac_phy_gpcs(dev->phy.mode)) {
2429                 /* XXX
2430                  * Make GPCS PHY address equal to EMAC index.
2431                  * We probably should take into account busy_phy_map
2432                  * and/or phy_map here.
2433                  *
2434                  * Note that the busy_phy_map is currently global
2435                  * while it should probably be per-ASIC...
2436                  */
2437                 dev->phy.gpcs_address = dev->gpcs_address;
2438                 if (dev->phy.gpcs_address == 0xffffffff)
2439                         dev->phy.address = dev->cell_index;
2440         }
2441
2442         emac_configure(dev);
2443
2444         if (dev->phy_address != 0xffffffff)
2445                 phy_map = ~(1 << dev->phy_address);
2446
2447         for (i = 0; i < 0x20; phy_map >>= 1, ++i)
2448                 if (!(phy_map & 1)) {
2449                         int r;
2450                         busy_phy_map |= 1 << i;
2451
2452                         /* Quick check if there is a PHY at the address */
2453                         r = emac_mdio_read(dev->ndev, i, MII_BMCR);
2454                         if (r == 0xffff || r < 0)
2455                                 continue;
2456                         if (!emac_mii_phy_probe(&dev->phy, i))
2457                                 break;
2458                 }
2459
2460         /* Enable external clock source */
2461 #ifdef CONFIG_PPC_DCR_NATIVE
2462         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2463                 dcri_clrset(SDR0, SDR0_MFR, SDR0_MFR_ECS, 0);
2464 #endif
2465         mutex_unlock(&emac_phy_map_lock);
2466         if (i == 0x20) {
2467                 printk(KERN_WARNING "%s: can't find PHY!\n", np->full_name);
2468                 return -ENXIO;
2469         }
2470
2471         /* Init PHY */
2472         if (dev->phy.def->ops->init)
2473                 dev->phy.def->ops->init(&dev->phy);
2474
2475         /* Disable any PHY features not supported by the platform */
2476         dev->phy.def->features &= ~dev->phy_feat_exc;
2477
2478         /* Setup initial link parameters */
2479         if (dev->phy.features & SUPPORTED_Autoneg) {
2480                 adv = dev->phy.features;
2481                 if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x))
2482                         adv |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
2483                 /* Restart autonegotiation */
2484                 dev->phy.def->ops->setup_aneg(&dev->phy, adv);
2485         } else {
2486                 u32 f = dev->phy.def->features;
2487                 int speed = SPEED_10, fd = DUPLEX_HALF;
2488
2489                 /* Select highest supported speed/duplex */
2490                 if (f & SUPPORTED_1000baseT_Full) {
2491                         speed = SPEED_1000;
2492                         fd = DUPLEX_FULL;
2493                 } else if (f & SUPPORTED_1000baseT_Half)
2494                         speed = SPEED_1000;
2495                 else if (f & SUPPORTED_100baseT_Full) {
2496                         speed = SPEED_100;
2497                         fd = DUPLEX_FULL;
2498                 } else if (f & SUPPORTED_100baseT_Half)
2499                         speed = SPEED_100;
2500                 else if (f & SUPPORTED_10baseT_Full)
2501                         fd = DUPLEX_FULL;
2502
2503                 /* Force link parameters */
2504                 dev->phy.def->ops->setup_forced(&dev->phy, speed, fd);
2505         }
2506         return 0;
2507 }
2508
2509 static int __devinit emac_init_config(struct emac_instance *dev)
2510 {
2511         struct device_node *np = dev->ofdev->node;
2512         const void *p;
2513         unsigned int plen;
2514         const char *pm, *phy_modes[] = {
2515                 [PHY_MODE_NA] = "",
2516                 [PHY_MODE_MII] = "mii",
2517                 [PHY_MODE_RMII] = "rmii",
2518                 [PHY_MODE_SMII] = "smii",
2519                 [PHY_MODE_RGMII] = "rgmii",
2520                 [PHY_MODE_TBI] = "tbi",
2521                 [PHY_MODE_GMII] = "gmii",
2522                 [PHY_MODE_RTBI] = "rtbi",
2523                 [PHY_MODE_SGMII] = "sgmii",
2524         };
2525
2526         /* Read config from device-tree */
2527         if (emac_read_uint_prop(np, "mal-device", &dev->mal_ph, 1))
2528                 return -ENXIO;
2529         if (emac_read_uint_prop(np, "mal-tx-channel", &dev->mal_tx_chan, 1))
2530                 return -ENXIO;
2531         if (emac_read_uint_prop(np, "mal-rx-channel", &dev->mal_rx_chan, 1))
2532                 return -ENXIO;
2533         if (emac_read_uint_prop(np, "cell-index", &dev->cell_index, 1))
2534                 return -ENXIO;
2535         if (emac_read_uint_prop(np, "max-frame-size", &dev->max_mtu, 0))
2536                 dev->max_mtu = 1500;
2537         if (emac_read_uint_prop(np, "rx-fifo-size", &dev->rx_fifo_size, 0))
2538                 dev->rx_fifo_size = 2048;
2539         if (emac_read_uint_prop(np, "tx-fifo-size", &dev->tx_fifo_size, 0))
2540                 dev->tx_fifo_size = 2048;
2541         if (emac_read_uint_prop(np, "rx-fifo-size-gige", &dev->rx_fifo_size_gige, 0))
2542                 dev->rx_fifo_size_gige = dev->rx_fifo_size;
2543         if (emac_read_uint_prop(np, "tx-fifo-size-gige", &dev->tx_fifo_size_gige, 0))
2544                 dev->tx_fifo_size_gige = dev->tx_fifo_size;
2545         if (emac_read_uint_prop(np, "phy-address", &dev->phy_address, 0))
2546                 dev->phy_address = 0xffffffff;
2547         if (emac_read_uint_prop(np, "phy-map", &dev->phy_map, 0))
2548                 dev->phy_map = 0xffffffff;
2549         if (emac_read_uint_prop(np, "gpcs-address", &dev->gpcs_address, 0))
2550                 dev->gpcs_address = 0xffffffff;
2551         if (emac_read_uint_prop(np->parent, "clock-frequency", &dev->opb_bus_freq, 1))
2552                 return -ENXIO;
2553         if (emac_read_uint_prop(np, "tah-device", &dev->tah_ph, 0))
2554                 dev->tah_ph = 0;
2555         if (emac_read_uint_prop(np, "tah-channel", &dev->tah_port, 0))
2556                 dev->tah_port = 0;
2557         if (emac_read_uint_prop(np, "mdio-device", &dev->mdio_ph, 0))
2558                 dev->mdio_ph = 0;
2559         if (emac_read_uint_prop(np, "zmii-device", &dev->zmii_ph, 0))
2560                 dev->zmii_ph = 0;;
2561         if (emac_read_uint_prop(np, "zmii-channel", &dev->zmii_port, 0))
2562                 dev->zmii_port = 0xffffffff;;
2563         if (emac_read_uint_prop(np, "rgmii-device", &dev->rgmii_ph, 0))
2564                 dev->rgmii_ph = 0;;
2565         if (emac_read_uint_prop(np, "rgmii-channel", &dev->rgmii_port, 0))
2566                 dev->rgmii_port = 0xffffffff;;
2567         if (emac_read_uint_prop(np, "fifo-entry-size", &dev->fifo_entry_size, 0))
2568                 dev->fifo_entry_size = 16;
2569         if (emac_read_uint_prop(np, "mal-burst-size", &dev->mal_burst_size, 0))
2570                 dev->mal_burst_size = 256;
2571
2572         /* PHY mode needs some decoding */
2573         dev->phy_mode = PHY_MODE_NA;
2574         pm = of_get_property(np, "phy-mode", &plen);
2575         if (pm != NULL) {
2576                 int i;
2577                 for (i = 0; i < ARRAY_SIZE(phy_modes); i++)
2578                         if (!strcasecmp(pm, phy_modes[i])) {
2579                                 dev->phy_mode = i;
2580                                 break;
2581                         }
2582         }
2583
2584         /* Backward compat with non-final DT */
2585         if (dev->phy_mode == PHY_MODE_NA && pm != NULL && plen == 4) {
2586                 u32 nmode = *(const u32 *)pm;
2587                 if (nmode > PHY_MODE_NA && nmode <= PHY_MODE_SGMII)
2588                         dev->phy_mode = nmode;
2589         }
2590
2591         /* Check EMAC version */
2592         if (of_device_is_compatible(np, "ibm,emac4sync")) {
2593                 dev->features |= (EMAC_FTR_EMAC4 | EMAC_FTR_EMAC4SYNC);
2594                 if (of_device_is_compatible(np, "ibm,emac-460ex") ||
2595                     of_device_is_compatible(np, "ibm,emac-460gt"))
2596                         dev->features |= EMAC_FTR_460EX_PHY_CLK_FIX;
2597         } else if (of_device_is_compatible(np, "ibm,emac4")) {
2598                 dev->features |= EMAC_FTR_EMAC4;
2599                 if (of_device_is_compatible(np, "ibm,emac-440gx"))
2600                         dev->features |= EMAC_FTR_440GX_PHY_CLK_FIX;
2601         } else {
2602                 if (of_device_is_compatible(np, "ibm,emac-440ep") ||
2603                     of_device_is_compatible(np, "ibm,emac-440gr"))
2604                         dev->features |= EMAC_FTR_440EP_PHY_CLK_FIX;
2605                 if (of_device_is_compatible(np, "ibm,emac-405ez")) {
2606 #ifdef CONFIG_IBM_NEW_EMAC_NO_FLOW_CTRL
2607                         dev->features |= EMAC_FTR_NO_FLOW_CONTROL_40x;
2608 #else
2609                         printk(KERN_ERR "%s: Flow control not disabled!\n",
2610                                         np->full_name);
2611                         return -ENXIO;
2612 #endif
2613                 }
2614
2615         }
2616
2617         /* Fixup some feature bits based on the device tree */
2618         if (of_get_property(np, "has-inverted-stacr-oc", NULL))
2619                 dev->features |= EMAC_FTR_STACR_OC_INVERT;
2620         if (of_get_property(np, "has-new-stacr-staopc", NULL))
2621                 dev->features |= EMAC_FTR_HAS_NEW_STACR;
2622
2623         /* CAB lacks the appropriate properties */
2624         if (of_device_is_compatible(np, "ibm,emac-axon"))
2625                 dev->features |= EMAC_FTR_HAS_NEW_STACR |
2626                         EMAC_FTR_STACR_OC_INVERT;
2627
2628         /* Enable TAH/ZMII/RGMII features as found */
2629         if (dev->tah_ph != 0) {
2630 #ifdef CONFIG_IBM_NEW_EMAC_TAH
2631                 dev->features |= EMAC_FTR_HAS_TAH;
2632 #else
2633                 printk(KERN_ERR "%s: TAH support not enabled !\n",
2634                        np->full_name);
2635                 return -ENXIO;
2636 #endif
2637         }
2638
2639         if (dev->zmii_ph != 0) {
2640 #ifdef CONFIG_IBM_NEW_EMAC_ZMII
2641                 dev->features |= EMAC_FTR_HAS_ZMII;
2642 #else
2643                 printk(KERN_ERR "%s: ZMII support not enabled !\n",
2644                        np->full_name);
2645                 return -ENXIO;
2646 #endif
2647         }
2648
2649         if (dev->rgmii_ph != 0) {
2650 #ifdef CONFIG_IBM_NEW_EMAC_RGMII
2651                 dev->features |= EMAC_FTR_HAS_RGMII;
2652 #else
2653                 printk(KERN_ERR "%s: RGMII support not enabled !\n",
2654                        np->full_name);
2655                 return -ENXIO;
2656 #endif
2657         }
2658
2659         /* Read MAC-address */
2660         p = of_get_property(np, "local-mac-address", NULL);
2661         if (p == NULL) {
2662                 printk(KERN_ERR "%s: Can't find local-mac-address property\n",
2663                        np->full_name);
2664                 return -ENXIO;
2665         }
2666         memcpy(dev->ndev->dev_addr, p, 6);
2667
2668         /* IAHT and GAHT filter parameterization */
2669         if (emac_has_feature(dev, EMAC_FTR_EMAC4SYNC)) {
2670                 dev->xaht_slots_shift = EMAC4SYNC_XAHT_SLOTS_SHIFT;
2671                 dev->xaht_width_shift = EMAC4SYNC_XAHT_WIDTH_SHIFT;
2672         } else {
2673                 dev->xaht_slots_shift = EMAC4_XAHT_SLOTS_SHIFT;
2674                 dev->xaht_width_shift = EMAC4_XAHT_WIDTH_SHIFT;
2675         }
2676
2677         DBG(dev, "features     : 0x%08x / 0x%08x\n", dev->features, EMAC_FTRS_POSSIBLE);
2678         DBG(dev, "tx_fifo_size : %d (%d gige)\n", dev->tx_fifo_size, dev->tx_fifo_size_gige);
2679         DBG(dev, "rx_fifo_size : %d (%d gige)\n", dev->rx_fifo_size, dev->rx_fifo_size_gige);
2680         DBG(dev, "max_mtu      : %d\n", dev->max_mtu);
2681         DBG(dev, "OPB freq     : %d\n", dev->opb_bus_freq);
2682
2683         return 0;
2684 }
2685
2686 static int __devinit emac_probe(struct of_device *ofdev,
2687                                 const struct of_device_id *match)
2688 {
2689         struct net_device *ndev;
2690         struct emac_instance *dev;
2691         struct device_node *np = ofdev->node;
2692         struct device_node **blist = NULL;
2693         int err, i;
2694
2695         /* Skip unused/unwired EMACS.  We leave the check for an unused
2696          * property here for now, but new flat device trees should set a
2697          * status property to "disabled" instead.
2698          */
2699         if (of_get_property(np, "unused", NULL) || !of_device_is_available(np))
2700                 return -ENODEV;
2701
2702         /* Find ourselves in the bootlist if we are there */
2703         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
2704                 if (emac_boot_list[i] == np)
2705                         blist = &emac_boot_list[i];
2706
2707         /* Allocate our net_device structure */
2708         err = -ENOMEM;
2709         ndev = alloc_etherdev(sizeof(struct emac_instance));
2710         if (!ndev) {
2711                 printk(KERN_ERR "%s: could not allocate ethernet device!\n",
2712                        np->full_name);
2713                 goto err_gone;
2714         }
2715         dev = netdev_priv(ndev);
2716         dev->ndev = ndev;
2717         dev->ofdev = ofdev;
2718         dev->blist = blist;
2719         SET_NETDEV_DEV(ndev, &ofdev->dev);
2720
2721         /* Initialize some embedded data structures */
2722         mutex_init(&dev->mdio_lock);
2723         mutex_init(&dev->link_lock);
2724         spin_lock_init(&dev->lock);
2725         INIT_WORK(&dev->reset_work, emac_reset_work);
2726
2727         /* Init various config data based on device-tree */
2728         err = emac_init_config(dev);
2729         if (err != 0)
2730                 goto err_free;
2731
2732         /* Get interrupts. EMAC irq is mandatory, WOL irq is optional */
2733         dev->emac_irq = irq_of_parse_and_map(np, 0);
2734         dev->wol_irq = irq_of_parse_and_map(np, 1);
2735         if (dev->emac_irq == NO_IRQ) {
2736                 printk(KERN_ERR "%s: Can't map main interrupt\n", np->full_name);
2737                 goto err_free;
2738         }
2739         ndev->irq = dev->emac_irq;
2740
2741         /* Map EMAC regs */
2742         if (of_address_to_resource(np, 0, &dev->rsrc_regs)) {
2743                 printk(KERN_ERR "%s: Can't get registers address\n",
2744                        np->full_name);
2745                 goto err_irq_unmap;
2746         }
2747         // TODO : request_mem_region
2748         dev->emacp = ioremap(dev->rsrc_regs.start,
2749                              dev->rsrc_regs.end - dev->rsrc_regs.start + 1);
2750         if (dev->emacp == NULL) {
2751                 printk(KERN_ERR "%s: Can't map device registers!\n",
2752                        np->full_name);
2753                 err = -ENOMEM;
2754                 goto err_irq_unmap;
2755         }
2756
2757         /* Wait for dependent devices */
2758         err = emac_wait_deps(dev);
2759         if (err) {
2760                 printk(KERN_ERR
2761                        "%s: Timeout waiting for dependent devices\n",
2762                        np->full_name);
2763                 /*  display more info about what's missing ? */
2764                 goto err_reg_unmap;
2765         }
2766         dev->mal = dev_get_drvdata(&dev->mal_dev->dev);
2767         if (dev->mdio_dev != NULL)
2768                 dev->mdio_instance = dev_get_drvdata(&dev->mdio_dev->dev);
2769
2770         /* Register with MAL */
2771         dev->commac.ops = &emac_commac_ops;
2772         dev->commac.dev = dev;
2773         dev->commac.tx_chan_mask = MAL_CHAN_MASK(dev->mal_tx_chan);
2774         dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan);
2775         err = mal_register_commac(dev->mal, &dev->commac);
2776         if (err) {
2777                 printk(KERN_ERR "%s: failed to register with mal %s!\n",
2778                        np->full_name, dev->mal_dev->node->full_name);
2779                 goto err_rel_deps;
2780         }
2781         dev->rx_skb_size = emac_rx_skb_size(ndev->mtu);
2782         dev->rx_sync_size = emac_rx_sync_size(ndev->mtu);
2783
2784         /* Get pointers to BD rings */
2785         dev->tx_desc =
2786             dev->mal->bd_virt + mal_tx_bd_offset(dev->mal, dev->mal_tx_chan);
2787         dev->rx_desc =
2788             dev->mal->bd_virt + mal_rx_bd_offset(dev->mal, dev->mal_rx_chan);
2789
2790         DBG(dev, "tx_desc %p" NL, dev->tx_desc);
2791         DBG(dev, "rx_desc %p" NL, dev->rx_desc);
2792
2793         /* Clean rings */
2794         memset(dev->tx_desc, 0, NUM_TX_BUFF * sizeof(struct mal_descriptor));
2795         memset(dev->rx_desc, 0, NUM_RX_BUFF * sizeof(struct mal_descriptor));
2796         memset(dev->tx_skb, 0, NUM_TX_BUFF * sizeof(struct sk_buff *));
2797         memset(dev->rx_skb, 0, NUM_RX_BUFF * sizeof(struct sk_buff *));
2798
2799         /* Attach to ZMII, if needed */
2800         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII) &&
2801             (err = zmii_attach(dev->zmii_dev, dev->zmii_port, &dev->phy_mode)) != 0)
2802                 goto err_unreg_commac;
2803
2804         /* Attach to RGMII, if needed */
2805         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII) &&
2806             (err = rgmii_attach(dev->rgmii_dev, dev->rgmii_port, dev->phy_mode)) != 0)
2807                 goto err_detach_zmii;
2808
2809         /* Attach to TAH, if needed */
2810         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
2811             (err = tah_attach(dev->tah_dev, dev->tah_port)) != 0)
2812                 goto err_detach_rgmii;
2813
2814         /* Set some link defaults before we can find out real parameters */
2815         dev->phy.speed = SPEED_100;
2816         dev->phy.duplex = DUPLEX_FULL;
2817         dev->phy.autoneg = AUTONEG_DISABLE;
2818         dev->phy.pause = dev->phy.asym_pause = 0;
2819         dev->stop_timeout = STOP_TIMEOUT_100;
2820         INIT_DELAYED_WORK(&dev->link_work, emac_link_timer);
2821
2822         /* Find PHY if any */
2823         err = emac_init_phy(dev);
2824         if (err != 0)
2825                 goto err_detach_tah;
2826
2827         /* Fill in the driver function table */
2828         ndev->open = &emac_open;
2829         if (dev->tah_dev)
2830                 ndev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
2831         ndev->tx_timeout = &emac_tx_timeout;
2832         ndev->watchdog_timeo = 5 * HZ;
2833         ndev->stop = &emac_close;
2834         ndev->get_stats = &emac_stats;
2835         ndev->set_multicast_list = &emac_set_multicast_list;
2836         ndev->do_ioctl = &emac_ioctl;
2837         if (emac_phy_supports_gige(dev->phy_mode)) {
2838                 ndev->hard_start_xmit = &emac_start_xmit_sg;
2839                 ndev->change_mtu = &emac_change_mtu;
2840                 dev->commac.ops = &emac_commac_sg_ops;
2841         } else {
2842                 ndev->hard_start_xmit = &emac_start_xmit;
2843         }
2844         SET_ETHTOOL_OPS(ndev, &emac_ethtool_ops);
2845
2846         netif_carrier_off(ndev);
2847         netif_stop_queue(ndev);
2848
2849         err = register_netdev(ndev);
2850         if (err) {
2851                 printk(KERN_ERR "%s: failed to register net device (%d)!\n",
2852                        np->full_name, err);
2853                 goto err_detach_tah;
2854         }
2855
2856         /* Set our drvdata last as we don't want them visible until we are
2857          * fully initialized
2858          */
2859         wmb();
2860         dev_set_drvdata(&ofdev->dev, dev);
2861
2862         /* There's a new kid in town ! Let's tell everybody */
2863         wake_up_all(&emac_probe_wait);
2864
2865
2866         printk(KERN_INFO "%s: EMAC-%d %s, MAC %pM\n",
2867                ndev->name, dev->cell_index, np->full_name, ndev->dev_addr);
2868
2869         if (dev->phy_mode == PHY_MODE_SGMII)
2870                 printk(KERN_NOTICE "%s: in SGMII mode\n", ndev->name);
2871
2872         if (dev->phy.address >= 0)
2873                 printk("%s: found %s PHY (0x%02x)\n", ndev->name,
2874                        dev->phy.def->name, dev->phy.address);
2875
2876         emac_dbg_register(dev);
2877
2878         /* Life is good */
2879         return 0;
2880
2881         /* I have a bad feeling about this ... */
2882
2883  err_detach_tah:
2884         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2885                 tah_detach(dev->tah_dev, dev->tah_port);
2886  err_detach_rgmii:
2887         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2888                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
2889  err_detach_zmii:
2890         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2891                 zmii_detach(dev->zmii_dev, dev->zmii_port);
2892  err_unreg_commac:
2893         mal_unregister_commac(dev->mal, &dev->commac);
2894  err_rel_deps:
2895         emac_put_deps(dev);
2896  err_reg_unmap:
2897         iounmap(dev->emacp);
2898  err_irq_unmap:
2899         if (dev->wol_irq != NO_IRQ)
2900                 irq_dispose_mapping(dev->wol_irq);
2901         if (dev->emac_irq != NO_IRQ)
2902                 irq_dispose_mapping(dev->emac_irq);
2903  err_free:
2904         kfree(ndev);
2905  err_gone:
2906         /* if we were on the bootlist, remove us as we won't show up and
2907          * wake up all waiters to notify them in case they were waiting
2908          * on us
2909          */
2910         if (blist) {
2911                 *blist = NULL;
2912                 wake_up_all(&emac_probe_wait);
2913         }
2914         return err;
2915 }
2916
2917 static int __devexit emac_remove(struct of_device *ofdev)
2918 {
2919         struct emac_instance *dev = dev_get_drvdata(&ofdev->dev);
2920
2921         DBG(dev, "remove" NL);
2922
2923         dev_set_drvdata(&ofdev->dev, NULL);
2924
2925         unregister_netdev(dev->ndev);
2926
2927         flush_scheduled_work();
2928
2929         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2930                 tah_detach(dev->tah_dev, dev->tah_port);
2931         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2932                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
2933         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2934                 zmii_detach(dev->zmii_dev, dev->zmii_port);
2935
2936         mal_unregister_commac(dev->mal, &dev->commac);
2937         emac_put_deps(dev);
2938
2939         emac_dbg_unregister(dev);
2940         iounmap(dev->emacp);
2941
2942         if (dev->wol_irq != NO_IRQ)
2943                 irq_dispose_mapping(dev->wol_irq);
2944         if (dev->emac_irq != NO_IRQ)
2945                 irq_dispose_mapping(dev->emac_irq);
2946
2947         kfree(dev->ndev);
2948
2949         return 0;
2950 }
2951
2952 /* XXX Features in here should be replaced by properties... */
2953 static struct of_device_id emac_match[] =
2954 {
2955         {
2956                 .type           = "network",
2957                 .compatible     = "ibm,emac",
2958         },
2959         {
2960                 .type           = "network",
2961                 .compatible     = "ibm,emac4",
2962         },
2963         {
2964                 .type           = "network",
2965                 .compatible     = "ibm,emac4sync",
2966         },
2967         {},
2968 };
2969
2970 static struct of_platform_driver emac_driver = {
2971         .name = "emac",
2972         .match_table = emac_match,
2973
2974         .probe = emac_probe,
2975         .remove = emac_remove,
2976 };
2977
2978 static void __init emac_make_bootlist(void)
2979 {
2980         struct device_node *np = NULL;
2981         int j, max, i = 0, k;
2982         int cell_indices[EMAC_BOOT_LIST_SIZE];
2983
2984         /* Collect EMACs */
2985         while((np = of_find_all_nodes(np)) != NULL) {
2986                 const u32 *idx;
2987
2988                 if (of_match_node(emac_match, np) == NULL)
2989                         continue;
2990                 if (of_get_property(np, "unused", NULL))
2991                         continue;
2992                 idx = of_get_property(np, "cell-index", NULL);
2993                 if (idx == NULL)
2994                         continue;
2995                 cell_indices[i] = *idx;
2996                 emac_boot_list[i++] = of_node_get(np);
2997                 if (i >= EMAC_BOOT_LIST_SIZE) {
2998                         of_node_put(np);
2999                         break;
3000                 }
3001         }
3002         max = i;
3003
3004         /* Bubble sort them (doh, what a creative algorithm :-) */
3005         for (i = 0; max > 1 && (i < (max - 1)); i++)
3006                 for (j = i; j < max; j++) {
3007                         if (cell_indices[i] > cell_indices[j]) {
3008                                 np = emac_boot_list[i];
3009                                 emac_boot_list[i] = emac_boot_list[j];
3010                                 emac_boot_list[j] = np;
3011                                 k = cell_indices[i];
3012                                 cell_indices[i] = cell_indices[j];
3013                                 cell_indices[j] = k;
3014                         }
3015                 }
3016 }
3017
3018 static int __init emac_init(void)
3019 {
3020         int rc;
3021
3022         printk(KERN_INFO DRV_DESC ", version " DRV_VERSION "\n");
3023
3024         /* Init debug stuff */
3025         emac_init_debug();
3026
3027         /* Build EMAC boot list */
3028         emac_make_bootlist();
3029
3030         /* Init submodules */
3031         rc = mal_init();
3032         if (rc)
3033                 goto err;
3034         rc = zmii_init();
3035         if (rc)
3036                 goto err_mal;
3037         rc = rgmii_init();
3038         if (rc)
3039                 goto err_zmii;
3040         rc = tah_init();
3041         if (rc)
3042                 goto err_rgmii;
3043         rc = of_register_platform_driver(&emac_driver);
3044         if (rc)
3045                 goto err_tah;
3046
3047         return 0;
3048
3049  err_tah:
3050         tah_exit();
3051  err_rgmii:
3052         rgmii_exit();
3053  err_zmii:
3054         zmii_exit();
3055  err_mal:
3056         mal_exit();
3057  err:
3058         return rc;
3059 }
3060
3061 static void __exit emac_exit(void)
3062 {
3063         int i;
3064
3065         of_unregister_platform_driver(&emac_driver);
3066
3067         tah_exit();
3068         rgmii_exit();
3069         zmii_exit();
3070         mal_exit();
3071         emac_fini_debug();
3072
3073         /* Destroy EMAC boot list */
3074         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
3075                 if (emac_boot_list[i])
3076                         of_node_put(emac_boot_list[i]);
3077 }
3078
3079 module_init(emac_init);
3080 module_exit(emac_exit);