drivers/net/ibm_newemac/core.c

   1 /*
   2  * drivers/net/ibm_newemac/core.c
   3  *
   4  * Driver for PowerPC 4xx on-chip ethernet controller.
   5  *
   6  * Copyright 2007 Benjamin Herrenschmidt, IBM Corp.
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * Based on the arch/ppc version of the driver:
  10  *
  11  * Copyright (c) 2004, 2005 Zultys Technologies.
  12  * Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>
  13  *
  14  * Based on original work by
  15  *      Matt Porter <mporter@kernel.crashing.org>
  16  *      (c) 2003 Benjamin Herrenschmidt <benh@kernel.crashing.org>
  17  *      Armin Kuster <akuster@mvista.com>
  18  *      Johnnie Peters <jpeters@mvista.com>
  19  *
  20  * This program is free software; you can redistribute  it and/or modify it
  21  * under  the terms of  the GNU General  Public License as published by the
  22  * Free Software Foundation;  either version 2 of the  License, or (at your
  23  * option) any later version.
  24  *
  25  */
  26
  27 #include <linux/sched.h>
  28 #include <linux/string.h>
  29 #include <linux/errno.h>
  30 #include <linux/delay.h>
  31 #include <linux/types.h>
  32 #include <linux/pci.h>
  33 #include <linux/etherdevice.h>
  34 #include <linux/skbuff.h>
  35 #include <linux/crc32.h>
  36 #include <linux/ethtool.h>
  37 #include <linux/mii.h>
  38 #include <linux/bitops.h>
  39 #include <linux/workqueue.h>
  40 #include <linux/of.h>
  41
  42 #include <asm/processor.h>
  43 #include <asm/io.h>
  44 #include <asm/dma.h>
  45 #include <asm/uaccess.h>
  46 #include <asm/dcr.h>
  47 #include <asm/dcr-regs.h>
  48
  49 #include "core.h"
  50
  51 /*
  52  * Lack of dma_unmap_???? calls is intentional.
  53  *
  54  * API-correct usage requires additional support state information to be
  55  * maintained for every RX and TX buffer descriptor (BD). Unfortunately, due to
  56  * EMAC design (e.g. TX buffer passed from network stack can be split into
  57  * several BDs, dma_map_single/dma_map_page can be used to map particular BD),
  58  * maintaining such information will add additional overhead.
  59  * Current DMA API implementation for 4xx processors only ensures cache coherency
  60  * and dma_unmap_???? routines are empty and are likely to stay this way.
  61  * I decided to omit dma_unmap_??? calls because I don't want to add additional
  62  * complexity just for the sake of following some abstract API, when it doesn't
  63  * add any real benefit to the driver. I understand that this decision maybe
  64  * controversial, but I really tried to make code API-correct and efficient
  65  * at the same time and didn't come up with code I liked :(.                --ebs
  66  */
  67
  68 #define DRV_NAME        "emac"
  69 #define DRV_VERSION     "3.54"
  70 #define DRV_DESC        "PPC 4xx OCP EMAC driver"
  71
  72 MODULE_DESCRIPTION(DRV_DESC);
  73 MODULE_AUTHOR
  74     ("Eugene Surovegin <eugene.surovegin@zultys.com> or <ebs@ebshome.net>");
  75 MODULE_LICENSE("GPL");
  76
  77 /*
  78  * PPC64 doesn't (yet) have a cacheable_memcpy
  79  */
  80 #ifdef CONFIG_PPC64
  81 #define cacheable_memcpy(d,s,n) memcpy((d),(s),(n))
  82 #endif
  83
  84 /* minimum number of free TX descriptors required to wake up TX process */
  85 #define EMAC_TX_WAKEUP_THRESH           (NUM_TX_BUFF / 4)
  86
  87 /* If packet size is less than this number, we allocate small skb and copy packet
  88  * contents into it instead of just sending original big skb up
  89  */
  90 #define EMAC_RX_COPY_THRESH             CONFIG_IBM_NEW_EMAC_RX_COPY_THRESHOLD
  91
  92 /* Since multiple EMACs share MDIO lines in various ways, we need
  93  * to avoid re-using the same PHY ID in cases where the arch didn't
  94  * setup precise phy_map entries
  95  *
  96  * XXX This is something that needs to be reworked as we can have multiple
  97  * EMAC "sets" (multiple ASICs containing several EMACs) though we can
  98  * probably require in that case to have explicit PHY IDs in the device-tree
  99  */
 100 static u32 busy_phy_map;
 101 static DEFINE_MUTEX(emac_phy_map_lock);
 102
 103 /* This is the wait queue used to wait on any event related to probe, that
 104  * is discovery of MALs, other EMACs, ZMII/RGMIIs, etc...
 105  */
 106 static DECLARE_WAIT_QUEUE_HEAD(emac_probe_wait);
 107
 108 /* Having stable interface names is a doomed idea. However, it would be nice
 109  * if we didn't have completely random interface names at boot too :-) It's
 110  * just a matter of making everybody's life easier. Since we are doing
 111  * threaded probing, it's a bit harder though. The base idea here is that
 112  * we make up a list of all emacs in the device-tree before we register the
 113  * driver. Every emac will then wait for the previous one in the list to
 114  * initialize before itself. We should also keep that list ordered by
 115  * cell_index.
 116  * That list is only 4 entries long, meaning that additional EMACs don't
 117  * get ordering guarantees unless EMAC_BOOT_LIST_SIZE is increased.
 118  */
 119
 120 #define EMAC_BOOT_LIST_SIZE     4
 121 static struct device_node *emac_boot_list[EMAC_BOOT_LIST_SIZE];
 122
 123 /* How long should I wait for dependent devices ? */
 124 #define EMAC_PROBE_DEP_TIMEOUT  (HZ * 5)
 125
 126 /* I don't want to litter system log with timeout errors
 127  * when we have brain-damaged PHY.
 128  */
 129 static inline void emac_report_timeout_error(struct emac_instance *dev,
 130                                              const char *error)
 131 {
 132         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX |
 133                                   EMAC_FTR_460EX_PHY_CLK_FIX |
 134                                   EMAC_FTR_440EP_PHY_CLK_FIX))
 135                 DBG(dev, "%s" NL, error);
 136         else if (net_ratelimit())
 137                 printk(KERN_ERR "%s: %s\n", dev->ofdev->node->full_name, error);
 138 }
 139
 140 /* EMAC PHY clock workaround:
 141  * 440EP/440GR has more sane SDR0_MFR register implementation than 440GX,
 142  * which allows controlling each EMAC clock
 143  */
 144 static inline void emac_rx_clk_tx(struct emac_instance *dev)
 145 {
 146 #ifdef CONFIG_PPC_DCR_NATIVE
 147         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 148                 dcri_clrset(SDR0, SDR0_MFR,
 149                             0, SDR0_MFR_ECS >> dev->cell_index);
 150 #endif
 151 }
 152
 153 static inline void emac_rx_clk_default(struct emac_instance *dev)
 154 {
 155 #ifdef CONFIG_PPC_DCR_NATIVE
 156         if (emac_has_feature(dev, EMAC_FTR_440EP_PHY_CLK_FIX))
 157                 dcri_clrset(SDR0, SDR0_MFR,
 158                             SDR0_MFR_ECS >> dev->cell_index, 0);
 159 #endif
 160 }
 161
 162 /* PHY polling intervals */
 163 #define PHY_POLL_LINK_ON        HZ
 164 #define PHY_POLL_LINK_OFF       (HZ / 5)
 165
 166 /* Graceful stop timeouts in us.
 167  * We should allow up to 1 frame time (full-duplex, ignoring collisions)
 168  */
 169 #define STOP_TIMEOUT_10         1230
 170 #define STOP_TIMEOUT_100        124
 171 #define STOP_TIMEOUT_1000       13
 172 #define STOP_TIMEOUT_1000_JUMBO 73
 173
 174 static unsigned char default_mcast_addr[] = {
 175         0x01, 0x80, 0xC2, 0x00, 0x00, 0x01
 176 };
 177
 178 /* Please, keep in sync with struct ibm_emac_stats/ibm_emac_error_stats */
 179 static const char emac_stats_keys[EMAC_ETHTOOL_STATS_COUNT][ETH_GSTRING_LEN] = {
 180         "rx_packets", "rx_bytes", "tx_packets", "tx_bytes", "rx_packets_csum",
 181         "tx_packets_csum", "tx_undo", "rx_dropped_stack", "rx_dropped_oom",
 182         "rx_dropped_error", "rx_dropped_resize", "rx_dropped_mtu",
 183         "rx_stopped", "rx_bd_errors", "rx_bd_overrun", "rx_bd_bad_packet",
 184         "rx_bd_runt_packet", "rx_bd_short_event", "rx_bd_alignment_error",
 185         "rx_bd_bad_fcs", "rx_bd_packet_too_long", "rx_bd_out_of_range",
 186         "rx_bd_in_range", "rx_parity", "rx_fifo_overrun", "rx_overrun",
 187         "rx_bad_packet", "rx_runt_packet", "rx_short_event",
 188         "rx_alignment_error", "rx_bad_fcs", "rx_packet_too_long",
 189         "rx_out_of_range", "rx_in_range", "tx_dropped", "tx_bd_errors",
 190         "tx_bd_bad_fcs", "tx_bd_carrier_loss", "tx_bd_excessive_deferral",
 191         "tx_bd_excessive_collisions", "tx_bd_late_collision",
 192         "tx_bd_multple_collisions", "tx_bd_single_collision",
 193         "tx_bd_underrun", "tx_bd_sqe", "tx_parity", "tx_underrun", "tx_sqe",
 194         "tx_errors"
 195 };
 196
 197 static irqreturn_t emac_irq(int irq, void *dev_instance);
 198 static void emac_clean_tx_ring(struct emac_instance *dev);
 199 static void __emac_set_multicast_list(struct emac_instance *dev);
 200
 201 static inline int emac_phy_supports_gige(int phy_mode)
 202 {
 203         return  phy_mode == PHY_MODE_GMII ||
 204                 phy_mode == PHY_MODE_RGMII ||
 205                 phy_mode == PHY_MODE_SGMII ||
 206                 phy_mode == PHY_MODE_TBI ||
 207                 phy_mode == PHY_MODE_RTBI;
 208 }
 209
 210 static inline int emac_phy_gpcs(int phy_mode)
 211 {
 212         return  phy_mode == PHY_MODE_SGMII ||
 213                 phy_mode == PHY_MODE_TBI ||
 214                 phy_mode == PHY_MODE_RTBI;
 215 }
 216
 217 static inline void emac_tx_enable(struct emac_instance *dev)
 218 {
 219         struct emac_regs __iomem *p = dev->emacp;
 220         u32 r;
 221
 222         DBG(dev, "tx_enable" NL);
 223
 224         r = in_be32(&p->mr0);
 225         if (!(r & EMAC_MR0_TXE))
 226                 out_be32(&p->mr0, r | EMAC_MR0_TXE);
 227 }
 228
 229 static void emac_tx_disable(struct emac_instance *dev)
 230 {
 231         struct emac_regs __iomem *p = dev->emacp;
 232         u32 r;
 233
 234         DBG(dev, "tx_disable" NL);
 235
 236         r = in_be32(&p->mr0);
 237         if (r & EMAC_MR0_TXE) {
 238                 int n = dev->stop_timeout;
 239                 out_be32(&p->mr0, r & ~EMAC_MR0_TXE);
 240                 while (!(in_be32(&p->mr0) & EMAC_MR0_TXI) && n) {
 241                         udelay(1);
 242                         --n;
 243                 }
 244                 if (unlikely(!n))
 245                         emac_report_timeout_error(dev, "TX disable timeout");
 246         }
 247 }
 248
 249 static void emac_rx_enable(struct emac_instance *dev)
 250 {
 251         struct emac_regs __iomem *p = dev->emacp;
 252         u32 r;
 253
 254         if (unlikely(test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags)))
 255                 goto out;
 256
 257         DBG(dev, "rx_enable" NL);
 258
 259         r = in_be32(&p->mr0);
 260         if (!(r & EMAC_MR0_RXE)) {
 261                 if (unlikely(!(r & EMAC_MR0_RXI))) {
 262                         /* Wait if previous async disable is still in progress */
 263                         int n = dev->stop_timeout;
 264                         while (!(r = in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 265                                 udelay(1);
 266                                 --n;
 267                         }
 268                         if (unlikely(!n))
 269                                 emac_report_timeout_error(dev,
 270                                                           "RX disable timeout");
 271                 }
 272                 out_be32(&p->mr0, r | EMAC_MR0_RXE);
 273         }
 274  out:
 275         ;
 276 }
 277
 278 static void emac_rx_disable(struct emac_instance *dev)
 279 {
 280         struct emac_regs __iomem *p = dev->emacp;
 281         u32 r;
 282
 283         DBG(dev, "rx_disable" NL);
 284
 285         r = in_be32(&p->mr0);
 286         if (r & EMAC_MR0_RXE) {
 287                 int n = dev->stop_timeout;
 288                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 289                 while (!(in_be32(&p->mr0) & EMAC_MR0_RXI) && n) {
 290                         udelay(1);
 291                         --n;
 292                 }
 293                 if (unlikely(!n))
 294                         emac_report_timeout_error(dev, "RX disable timeout");
 295         }
 296 }
 297
 298 static inline void emac_netif_stop(struct emac_instance *dev)
 299 {
 300         netif_tx_lock_bh(dev->ndev);
 301         netif_addr_lock(dev->ndev);
 302         dev->no_mcast = 1;
 303         netif_addr_unlock(dev->ndev);
 304         netif_tx_unlock_bh(dev->ndev);
 305         dev->ndev->trans_start = jiffies;       /* prevent tx timeout */
 306         mal_poll_disable(dev->mal, &dev->commac);
 307         netif_tx_disable(dev->ndev);
 308 }
 309
 310 static inline void emac_netif_start(struct emac_instance *dev)
 311 {
 312         netif_tx_lock_bh(dev->ndev);
 313         netif_addr_lock(dev->ndev);
 314         dev->no_mcast = 0;
 315         if (dev->mcast_pending && netif_running(dev->ndev))
 316                 __emac_set_multicast_list(dev);
 317         netif_addr_unlock(dev->ndev);
 318         netif_tx_unlock_bh(dev->ndev);
 319
 320         netif_wake_queue(dev->ndev);
 321
 322         /* NOTE: unconditional netif_wake_queue is only appropriate
 323          * so long as all callers are assured to have free tx slots
 324          * (taken from tg3... though the case where that is wrong is
 325          *  not terribly harmful)
 326          */
 327         mal_poll_enable(dev->mal, &dev->commac);
 328 }
 329
 330 static inline void emac_rx_disable_async(struct emac_instance *dev)
 331 {
 332         struct emac_regs __iomem *p = dev->emacp;
 333         u32 r;
 334
 335         DBG(dev, "rx_disable_async" NL);
 336
 337         r = in_be32(&p->mr0);
 338         if (r & EMAC_MR0_RXE)
 339                 out_be32(&p->mr0, r & ~EMAC_MR0_RXE);
 340 }
 341
 342 static int emac_reset(struct emac_instance *dev)
 343 {
 344         struct emac_regs __iomem *p = dev->emacp;
 345         int n = 20;
 346
 347         DBG(dev, "reset" NL);
 348
 349         if (!dev->reset_failed) {
 350                 /* 40x erratum suggests stopping RX channel before reset,
 351                  * we stop TX as well
 352                  */
 353                 emac_rx_disable(dev);
 354                 emac_tx_disable(dev);
 355         }
 356
 357 #ifdef CONFIG_PPC_DCR_NATIVE
 358         /* Enable internal clock source */
 359         if (emac_has_feature(dev, EMAC_FTR_460EX_PHY_CLK_FIX))
 360                 dcri_clrset(SDR0, SDR0_ETH_CFG,
 361                             0, SDR0_ETH_CFG_ECS << dev->cell_index);
 362 #endif
 363
 364         out_be32(&p->mr0, EMAC_MR0_SRST);
 365         while ((in_be32(&p->mr0) & EMAC_MR0_SRST) && n)
 366                 --n;
 367
 368 #ifdef CONFIG_PPC_DCR_NATIVE
 369          /* Enable external clock source */
 370         if (emac_has_feature(dev, EMAC_FTR_460EX_PHY_CLK_FIX))
 371                 dcri_clrset(SDR0, SDR0_ETH_CFG,
 372                             SDR0_ETH_CFG_ECS << dev->cell_index, 0);
 373 #endif
 374
 375         if (n) {
 376                 dev->reset_failed = 0;
 377                 return 0;
 378         } else {
 379                 emac_report_timeout_error(dev, "reset timeout");
 380                 dev->reset_failed = 1;
 381                 return -ETIMEDOUT;
 382         }
 383 }
 384
 385 static void emac_hash_mc(struct emac_instance *dev)
 386 {
 387         const int regs = EMAC_XAHT_REGS(dev);
 388         u32 *gaht_base = emac_gaht_base(dev);
 389         u32 gaht_temp[regs];
 390         struct dev_mc_list *dmi;
 391         int i;
 392
 393         DBG(dev, "hash_mc %d" NL, dev->ndev->mc_count);
 394
 395         memset(gaht_temp, 0, sizeof (gaht_temp));
 396
 397         for (dmi = dev->ndev->mc_list; dmi; dmi = dmi->next) {
 398                 int slot, reg, mask;
 399                 DBG2(dev, "mc %pM" NL, dmi->dmi_addr);
 400
 401                 slot = EMAC_XAHT_CRC_TO_SLOT(dev, ether_crc(ETH_ALEN, dmi->dmi_addr));
 402                 reg = EMAC_XAHT_SLOT_TO_REG(dev, slot);
 403                 mask = EMAC_XAHT_SLOT_TO_MASK(dev, slot);
 404
 405                 gaht_temp[reg] |= mask;
 406         }
 407
 408         for (i = 0; i < regs; i++)
 409                 out_be32(gaht_base + i, gaht_temp[i]);
 410 }
 411
 412 static inline u32 emac_iff2rmr(struct net_device *ndev)
 413 {
 414         struct emac_instance *dev = netdev_priv(ndev);
 415         u32 r;
 416
 417         r = EMAC_RMR_SP | EMAC_RMR_SFCS | EMAC_RMR_IAE | EMAC_RMR_BAE;
 418
 419         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 420             r |= EMAC4_RMR_BASE;
 421         else
 422             r |= EMAC_RMR_BASE;
 423
 424         if (ndev->flags & IFF_PROMISC)
 425                 r |= EMAC_RMR_PME;
 426         else if (ndev->flags & IFF_ALLMULTI ||
 427                          (ndev->mc_count > EMAC_XAHT_SLOTS(dev)))
 428                 r |= EMAC_RMR_PMME;
 429         else if (ndev->mc_count > 0)
 430                 r |= EMAC_RMR_MAE;
 431
 432         return r;
 433 }
 434
 435 static u32 __emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 436 {
 437         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC_MR1_TR0_MULT;
 438
 439         DBG2(dev, "__emac_calc_base_mr1" NL);
 440
 441         switch(tx_size) {
 442         case 2048:
 443                 ret |= EMAC_MR1_TFS_2K;
 444                 break;
 445         default:
 446                 printk(KERN_WARNING "%s: Unknown Tx FIFO size %d\n",
 447                        dev->ndev->name, tx_size);
 448         }
 449
 450         switch(rx_size) {
 451         case 16384:
 452                 ret |= EMAC_MR1_RFS_16K;
 453                 break;
 454         case 4096:
 455                 ret |= EMAC_MR1_RFS_4K;
 456                 break;
 457         default:
 458                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 459                        dev->ndev->name, rx_size);
 460         }
 461
 462         return ret;
 463 }
 464
 465 static u32 __emac4_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 466 {
 467         u32 ret = EMAC_MR1_VLE | EMAC_MR1_IST | EMAC4_MR1_TR |
 468                 EMAC4_MR1_OBCI(dev->opb_bus_freq / 1000000);
 469
 470         DBG2(dev, "__emac4_calc_base_mr1" NL);
 471
 472         switch(tx_size) {
 473         case 16384:
 474                 ret |= EMAC4_MR1_TFS_16K;
 475                 break;
 476         case 4096:
 477                 ret |= EMAC4_MR1_TFS_4K;
 478                 break;
 479         case 2048:
 480                 ret |= EMAC4_MR1_TFS_2K;
 481                 break;
 482         default:
 483                 printk(KERN_WARNING "%s: Unknown Tx FIFO size %d\n",
 484                        dev->ndev->name, tx_size);
 485         }
 486
 487         switch(rx_size) {
 488         case 16384:
 489                 ret |= EMAC4_MR1_RFS_16K;
 490                 break;
 491         case 4096:
 492                 ret |= EMAC4_MR1_RFS_4K;
 493                 break;
 494         case 2048:
 495                 ret |= EMAC4_MR1_RFS_2K;
 496                 break;
 497         default:
 498                 printk(KERN_WARNING "%s: Unknown Rx FIFO size %d\n",
 499                        dev->ndev->name, rx_size);
 500         }
 501
 502         return ret;
 503 }
 504
 505 static u32 emac_calc_base_mr1(struct emac_instance *dev, int tx_size, int rx_size)
 506 {
 507         return emac_has_feature(dev, EMAC_FTR_EMAC4) ?
 508                 __emac4_calc_base_mr1(dev, tx_size, rx_size) :
 509                 __emac_calc_base_mr1(dev, tx_size, rx_size);
 510 }
 511
 512 static inline u32 emac_calc_trtr(struct emac_instance *dev, unsigned int size)
 513 {
 514         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 515                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT_EMAC4;
 516         else
 517                 return ((size >> 6) - 1) << EMAC_TRTR_SHIFT;
 518 }
 519
 520 static inline u32 emac_calc_rwmr(struct emac_instance *dev,
 521                                  unsigned int low, unsigned int high)
 522 {
 523         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 524                 return (low << 22) | ( (high & 0x3ff) << 6);
 525         else
 526                 return (low << 23) | ( (high & 0x1ff) << 7);
 527 }
 528
 529 static int emac_configure(struct emac_instance *dev)
 530 {
 531         struct emac_regs __iomem *p = dev->emacp;
 532         struct net_device *ndev = dev->ndev;
 533         int tx_size, rx_size, link = netif_carrier_ok(dev->ndev);
 534         u32 r, mr1 = 0;
 535
 536         DBG(dev, "configure" NL);
 537
 538         if (!link) {
 539                 out_be32(&p->mr1, in_be32(&p->mr1)
 540                          | EMAC_MR1_FDE | EMAC_MR1_ILE);
 541                 udelay(100);
 542         } else if (emac_reset(dev) < 0)
 543                 return -ETIMEDOUT;
 544
 545         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
 546                 tah_reset(dev->tah_dev);
 547
 548         DBG(dev, " link = %d duplex = %d, pause = %d, asym_pause = %d\n",
 549             link, dev->phy.duplex, dev->phy.pause, dev->phy.asym_pause);
 550
 551         /* Default fifo sizes */
 552         tx_size = dev->tx_fifo_size;
 553         rx_size = dev->rx_fifo_size;
 554
 555         /* No link, force loopback */
 556         if (!link)
 557                 mr1 = EMAC_MR1_FDE | EMAC_MR1_ILE;
 558
 559         /* Check for full duplex */
 560         else if (dev->phy.duplex == DUPLEX_FULL)
 561                 mr1 |= EMAC_MR1_FDE | EMAC_MR1_MWSW_001;
 562
 563         /* Adjust fifo sizes, mr1 and timeouts based on link speed */
 564         dev->stop_timeout = STOP_TIMEOUT_10;
 565         switch (dev->phy.speed) {
 566         case SPEED_1000:
 567                 if (emac_phy_gpcs(dev->phy.mode)) {
 568                         mr1 |= EMAC_MR1_MF_1000GPCS | EMAC_MR1_MF_IPPA(
 569                                 (dev->phy.gpcs_address != 0xffffffff) ?
 570                                  dev->phy.gpcs_address : dev->phy.address);
 571
 572                         /* Put some arbitrary OUI, Manuf & Rev IDs so we can
 573                          * identify this GPCS PHY later.
 574                          */
 575                         out_be32(&p->u1.emac4.ipcr, 0xdeadbeef);
 576                 } else
 577                         mr1 |= EMAC_MR1_MF_1000;
 578
 579                 /* Extended fifo sizes */
 580                 tx_size = dev->tx_fifo_size_gige;
 581                 rx_size = dev->rx_fifo_size_gige;
 582
 583                 if (dev->ndev->mtu > ETH_DATA_LEN) {
 584                         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 585                                 mr1 |= EMAC4_MR1_JPSM;
 586                         else
 587                                 mr1 |= EMAC_MR1_JPSM;
 588                         dev->stop_timeout = STOP_TIMEOUT_1000_JUMBO;
 589                 } else
 590                         dev->stop_timeout = STOP_TIMEOUT_1000;
 591                 break;
 592         case SPEED_100:
 593                 mr1 |= EMAC_MR1_MF_100;
 594                 dev->stop_timeout = STOP_TIMEOUT_100;
 595                 break;
 596         default: /* make gcc happy */
 597                 break;
 598         }
 599
 600         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 601                 rgmii_set_speed(dev->rgmii_dev, dev->rgmii_port,
 602                                 dev->phy.speed);
 603         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 604                 zmii_set_speed(dev->zmii_dev, dev->zmii_port, dev->phy.speed);
 605
 606         /* on 40x erratum forces us to NOT use integrated flow control,
 607          * let's hope it works on 44x ;)
 608          */
 609         if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x) &&
 610             dev->phy.duplex == DUPLEX_FULL) {
 611                 if (dev->phy.pause)
 612                         mr1 |= EMAC_MR1_EIFC | EMAC_MR1_APP;
 613                 else if (dev->phy.asym_pause)
 614                         mr1 |= EMAC_MR1_APP;
 615         }
 616
 617         /* Add base settings & fifo sizes & program MR1 */
 618         mr1 |= emac_calc_base_mr1(dev, tx_size, rx_size);
 619         out_be32(&p->mr1, mr1);
 620
 621         /* Set individual MAC address */
 622         out_be32(&p->iahr, (ndev->dev_addr[0] << 8) | ndev->dev_addr[1]);
 623         out_be32(&p->ialr, (ndev->dev_addr[2] << 24) |
 624                  (ndev->dev_addr[3] << 16) | (ndev->dev_addr[4] << 8) |
 625                  ndev->dev_addr[5]);
 626
 627         /* VLAN Tag Protocol ID */
 628         out_be32(&p->vtpid, 0x8100);
 629
 630         /* Receive mode register */
 631         r = emac_iff2rmr(ndev);
 632         if (r & EMAC_RMR_MAE)
 633                 emac_hash_mc(dev);
 634         out_be32(&p->rmr, r);
 635
 636         /* FIFOs thresholds */
 637         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 638                 r = EMAC4_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 639                                tx_size / 2 / dev->fifo_entry_size);
 640         else
 641                 r = EMAC_TMR1((dev->mal_burst_size / dev->fifo_entry_size) + 1,
 642                               tx_size / 2 / dev->fifo_entry_size);
 643         out_be32(&p->tmr1, r);
 644         out_be32(&p->trtr, emac_calc_trtr(dev, tx_size / 2));
 645
 646         /* PAUSE frame is sent when RX FIFO reaches its high-water mark,
 647            there should be still enough space in FIFO to allow the our link
 648            partner time to process this frame and also time to send PAUSE
 649            frame itself.
 650
 651            Here is the worst case scenario for the RX FIFO "headroom"
 652            (from "The Switch Book") (100Mbps, without preamble, inter-frame gap):
 653
 654            1) One maximum-length frame on TX                    1522 bytes
 655            2) One PAUSE frame time                                64 bytes
 656            3) PAUSE frame decode time allowance                   64 bytes
 657            4) One maximum-length frame on RX                    1522 bytes
 658            5) Round-trip propagation delay of the link (100Mb)    15 bytes
 659            ----------
 660            3187 bytes
 661
 662            I chose to set high-water mark to RX_FIFO_SIZE / 4 (1024 bytes)
 663            low-water mark  to RX_FIFO_SIZE / 8 (512 bytes)
 664          */
 665         r = emac_calc_rwmr(dev, rx_size / 8 / dev->fifo_entry_size,
 666                            rx_size / 4 / dev->fifo_entry_size);
 667         out_be32(&p->rwmr, r);
 668
 669         /* Set PAUSE timer to the maximum */
 670         out_be32(&p->ptr, 0xffff);
 671
 672         /* IRQ sources */
 673         r = EMAC_ISR_OVR | EMAC_ISR_BP | EMAC_ISR_SE |
 674                 EMAC_ISR_ALE | EMAC_ISR_BFCS | EMAC_ISR_PTLE | EMAC_ISR_ORE |
 675                 EMAC_ISR_IRE | EMAC_ISR_TE;
 676         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 677             r |= EMAC4_ISR_TXPE | EMAC4_ISR_RXPE /* | EMAC4_ISR_TXUE |
 678                                                   EMAC4_ISR_RXOE | */;
 679         out_be32(&p->iser,  r);
 680
 681         /* We need to take GPCS PHY out of isolate mode after EMAC reset */
 682         if (emac_phy_gpcs(dev->phy.mode)) {
 683                 if (dev->phy.gpcs_address != 0xffffffff)
 684                         emac_mii_reset_gpcs(&dev->phy);
 685                 else
 686                         emac_mii_reset_phy(&dev->phy);
 687         }
 688
 689         return 0;
 690 }
 691
 692 static void emac_reinitialize(struct emac_instance *dev)
 693 {
 694         DBG(dev, "reinitialize" NL);
 695
 696         emac_netif_stop(dev);
 697         if (!emac_configure(dev)) {
 698                 emac_tx_enable(dev);
 699                 emac_rx_enable(dev);
 700         }
 701         emac_netif_start(dev);
 702 }
 703
 704 static void emac_full_tx_reset(struct emac_instance *dev)
 705 {
 706         DBG(dev, "full_tx_reset" NL);
 707
 708         emac_tx_disable(dev);
 709         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
 710         emac_clean_tx_ring(dev);
 711         dev->tx_cnt = dev->tx_slot = dev->ack_slot = 0;
 712
 713         emac_configure(dev);
 714
 715         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
 716         emac_tx_enable(dev);
 717         emac_rx_enable(dev);
 718 }
 719
 720 static void emac_reset_work(struct work_struct *work)
 721 {
 722         struct emac_instance *dev = container_of(work, struct emac_instance, reset_work);
 723
 724         DBG(dev, "reset_work" NL);
 725
 726         mutex_lock(&dev->link_lock);
 727         if (dev->opened) {
 728                 emac_netif_stop(dev);
 729                 emac_full_tx_reset(dev);
 730                 emac_netif_start(dev);
 731         }
 732         mutex_unlock(&dev->link_lock);
 733 }
 734
 735 static void emac_tx_timeout(struct net_device *ndev)
 736 {
 737         struct emac_instance *dev = netdev_priv(ndev);
 738
 739         DBG(dev, "tx_timeout" NL);
 740
 741         schedule_work(&dev->reset_work);
 742 }
 743
 744
 745 static inline int emac_phy_done(struct emac_instance *dev, u32 stacr)
 746 {
 747         int done = !!(stacr & EMAC_STACR_OC);
 748
 749         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 750                 done = !done;
 751
 752         return done;
 753 };
 754
 755 static int __emac_mdio_read(struct emac_instance *dev, u8 id, u8 reg)
 756 {
 757         struct emac_regs __iomem *p = dev->emacp;
 758         u32 r = 0;
 759         int n, err = -ETIMEDOUT;
 760
 761         mutex_lock(&dev->mdio_lock);
 762
 763         DBG2(dev, "mdio_read(%02x,%02x)" NL, id, reg);
 764
 765         /* Enable proper MDIO port */
 766         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 767                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 768         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 769                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 770
 771         /* Wait for management interface to become idle */
 772         n = 20;
 773         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 774                 udelay(1);
 775                 if (!--n) {
 776                         DBG2(dev, " -> timeout wait idle\n");
 777                         goto bail;
 778                 }
 779         }
 780
 781         /* Issue read command */
 782         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 783                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 784         else
 785                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 786         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 787                 r |= EMAC_STACR_OC;
 788         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 789                 r |= EMACX_STACR_STAC_READ;
 790         else
 791                 r |= EMAC_STACR_STAC_READ;
 792         r |= (reg & EMAC_STACR_PRA_MASK)
 793                 | ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT);
 794         out_be32(&p->stacr, r);
 795
 796         /* Wait for read to complete */
 797         n = 200;
 798         while (!emac_phy_done(dev, (r = in_be32(&p->stacr)))) {
 799                 udelay(1);
 800                 if (!--n) {
 801                         DBG2(dev, " -> timeout wait complete\n");
 802                         goto bail;
 803                 }
 804         }
 805
 806         if (unlikely(r & EMAC_STACR_PHYE)) {
 807                 DBG(dev, "mdio_read(%02x, %02x) failed" NL, id, reg);
 808                 err = -EREMOTEIO;
 809                 goto bail;
 810         }
 811
 812         r = ((r >> EMAC_STACR_PHYD_SHIFT) & EMAC_STACR_PHYD_MASK);
 813
 814         DBG2(dev, "mdio_read -> %04x" NL, r);
 815         err = 0;
 816  bail:
 817         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 818                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 819         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 820                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 821         mutex_unlock(&dev->mdio_lock);
 822
 823         return err == 0 ? r : err;
 824 }
 825
 826 static void __emac_mdio_write(struct emac_instance *dev, u8 id, u8 reg,
 827                               u16 val)
 828 {
 829         struct emac_regs __iomem *p = dev->emacp;
 830         u32 r = 0;
 831         int n, err = -ETIMEDOUT;
 832
 833         mutex_lock(&dev->mdio_lock);
 834
 835         DBG2(dev, "mdio_write(%02x,%02x,%04x)" NL, id, reg, val);
 836
 837         /* Enable proper MDIO port */
 838         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 839                 zmii_get_mdio(dev->zmii_dev, dev->zmii_port);
 840         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 841                 rgmii_get_mdio(dev->rgmii_dev, dev->rgmii_port);
 842
 843         /* Wait for management interface to be idle */
 844         n = 20;
 845         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 846                 udelay(1);
 847                 if (!--n) {
 848                         DBG2(dev, " -> timeout wait idle\n");
 849                         goto bail;
 850                 }
 851         }
 852
 853         /* Issue write command */
 854         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
 855                 r = EMAC4_STACR_BASE(dev->opb_bus_freq);
 856         else
 857                 r = EMAC_STACR_BASE(dev->opb_bus_freq);
 858         if (emac_has_feature(dev, EMAC_FTR_STACR_OC_INVERT))
 859                 r |= EMAC_STACR_OC;
 860         if (emac_has_feature(dev, EMAC_FTR_HAS_NEW_STACR))
 861                 r |= EMACX_STACR_STAC_WRITE;
 862         else
 863                 r |= EMAC_STACR_STAC_WRITE;
 864         r |= (reg & EMAC_STACR_PRA_MASK) |
 865                 ((id & EMAC_STACR_PCDA_MASK) << EMAC_STACR_PCDA_SHIFT) |
 866                 (val << EMAC_STACR_PHYD_SHIFT);
 867         out_be32(&p->stacr, r);
 868
 869         /* Wait for write to complete */
 870         n = 200;
 871         while (!emac_phy_done(dev, in_be32(&p->stacr))) {
 872                 udelay(1);
 873                 if (!--n) {
 874                         DBG2(dev, " -> timeout wait complete\n");
 875                         goto bail;
 876                 }
 877         }
 878         err = 0;
 879  bail:
 880         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
 881                 rgmii_put_mdio(dev->rgmii_dev, dev->rgmii_port);
 882         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
 883                 zmii_put_mdio(dev->zmii_dev, dev->zmii_port);
 884         mutex_unlock(&dev->mdio_lock);
 885 }
 886
 887 static int emac_mdio_read(struct net_device *ndev, int id, int reg)
 888 {
 889         struct emac_instance *dev = netdev_priv(ndev);
 890         int res;
 891
 892         res = __emac_mdio_read((dev->mdio_instance &&
 893                                 dev->phy.gpcs_address != id) ?
 894                                 dev->mdio_instance : dev,
 895                                (u8) id, (u8) reg);
 896         return res;
 897 }
 898
 899 static void emac_mdio_write(struct net_device *ndev, int id, int reg, int val)
 900 {
 901         struct emac_instance *dev = netdev_priv(ndev);
 902
 903         __emac_mdio_write((dev->mdio_instance &&
 904                            dev->phy.gpcs_address != id) ?
 905                            dev->mdio_instance : dev,
 906                           (u8) id, (u8) reg, (u16) val);
 907 }
 908
 909 /* Tx lock BH */
 910 static void __emac_set_multicast_list(struct emac_instance *dev)
 911 {
 912         struct emac_regs __iomem *p = dev->emacp;
 913         u32 rmr = emac_iff2rmr(dev->ndev);
 914
 915         DBG(dev, "__multicast %08x" NL, rmr);
 916
 917         /* I decided to relax register access rules here to avoid
 918          * full EMAC reset.
 919          *
 920          * There is a real problem with EMAC4 core if we use MWSW_001 bit
 921          * in MR1 register and do a full EMAC reset.
 922          * One TX BD status update is delayed and, after EMAC reset, it
 923          * never happens, resulting in TX hung (it'll be recovered by TX
 924          * timeout handler eventually, but this is just gross).
 925          * So we either have to do full TX reset or try to cheat here :)
 926          *
 927          * The only required change is to RX mode register, so I *think* all
 928          * we need is just to stop RX channel. This seems to work on all
 929          * tested SoCs.                                                --ebs
 930          *
 931          * If we need the full reset, we might just trigger the workqueue
 932          * and do it async... a bit nasty but should work --BenH
 933          */
 934         dev->mcast_pending = 0;
 935         emac_rx_disable(dev);
 936         if (rmr & EMAC_RMR_MAE)
 937                 emac_hash_mc(dev);
 938         out_be32(&p->rmr, rmr);
 939         emac_rx_enable(dev);
 940 }
 941
 942 /* Tx lock BH */
 943 static void emac_set_multicast_list(struct net_device *ndev)
 944 {
 945         struct emac_instance *dev = netdev_priv(ndev);
 946
 947         DBG(dev, "multicast" NL);
 948
 949         BUG_ON(!netif_running(dev->ndev));
 950
 951         if (dev->no_mcast) {
 952                 dev->mcast_pending = 1;
 953                 return;
 954         }
 955         __emac_set_multicast_list(dev);
 956 }
 957
 958 static int emac_resize_rx_ring(struct emac_instance *dev, int new_mtu)
 959 {
 960         int rx_sync_size = emac_rx_sync_size(new_mtu);
 961         int rx_skb_size = emac_rx_skb_size(new_mtu);
 962         int i, ret = 0;
 963
 964         mutex_lock(&dev->link_lock);
 965         emac_netif_stop(dev);
 966         emac_rx_disable(dev);
 967         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
 968
 969         if (dev->rx_sg_skb) {
 970                 ++dev->estats.rx_dropped_resize;
 971                 dev_kfree_skb(dev->rx_sg_skb);
 972                 dev->rx_sg_skb = NULL;
 973         }
 974
 975         /* Make a first pass over RX ring and mark BDs ready, dropping
 976          * non-processed packets on the way. We need this as a separate pass
 977          * to simplify error recovery in the case of allocation failure later.
 978          */
 979         for (i = 0; i < NUM_RX_BUFF; ++i) {
 980                 if (dev->rx_desc[i].ctrl & MAL_RX_CTRL_FIRST)
 981                         ++dev->estats.rx_dropped_resize;
 982
 983                 dev->rx_desc[i].data_len = 0;
 984                 dev->rx_desc[i].ctrl = MAL_RX_CTRL_EMPTY |
 985                     (i == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
 986         }
 987
 988         /* Reallocate RX ring only if bigger skb buffers are required */
 989         if (rx_skb_size <= dev->rx_skb_size)
 990                 goto skip;
 991
 992         /* Second pass, allocate new skbs */
 993         for (i = 0; i < NUM_RX_BUFF; ++i) {
 994                 struct sk_buff *skb = alloc_skb(rx_skb_size, GFP_ATOMIC);
 995                 if (!skb) {
 996                         ret = -ENOMEM;
 997                         goto oom;
 998                 }
 999
1000                 BUG_ON(!dev->rx_skb[i]);
1001                 dev_kfree_skb(dev->rx_skb[i]);
1002
1003                 skb_reserve(skb, EMAC_RX_SKB_HEADROOM + 2);
1004                 dev->rx_desc[i].data_ptr =
1005                     dma_map_single(&dev->ofdev->dev, skb->data - 2, rx_sync_size,
1006                                    DMA_FROM_DEVICE) + 2;
1007                 dev->rx_skb[i] = skb;
1008         }
1009  skip:
1010         /* Check if we need to change "Jumbo" bit in MR1 */
1011         if ((new_mtu > ETH_DATA_LEN) ^ (dev->ndev->mtu > ETH_DATA_LEN)) {
1012                 /* This is to prevent starting RX channel in emac_rx_enable() */
1013                 set_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1014
1015                 dev->ndev->mtu = new_mtu;
1016                 emac_full_tx_reset(dev);
1017         }
1018
1019         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(new_mtu));
1020  oom:
1021         /* Restart RX */
1022         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1023         dev->rx_slot = 0;
1024         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1025         emac_rx_enable(dev);
1026         emac_netif_start(dev);
1027         mutex_unlock(&dev->link_lock);
1028
1029         return ret;
1030 }
1031
1032 /* Process ctx, rtnl_lock semaphore */
1033 static int emac_change_mtu(struct net_device *ndev, int new_mtu)
1034 {
1035         struct emac_instance *dev = netdev_priv(ndev);
1036         int ret = 0;
1037
1038         if (new_mtu < EMAC_MIN_MTU || new_mtu > dev->max_mtu)
1039                 return -EINVAL;
1040
1041         DBG(dev, "change_mtu(%d)" NL, new_mtu);
1042
1043         if (netif_running(ndev)) {
1044                 /* Check if we really need to reinitalize RX ring */
1045                 if (emac_rx_skb_size(ndev->mtu) != emac_rx_skb_size(new_mtu))
1046                         ret = emac_resize_rx_ring(dev, new_mtu);
1047         }
1048
1049         if (!ret) {
1050                 ndev->mtu = new_mtu;
1051                 dev->rx_skb_size = emac_rx_skb_size(new_mtu);
1052                 dev->rx_sync_size = emac_rx_sync_size(new_mtu);
1053         }
1054
1055         return ret;
1056 }
1057
1058 static void emac_clean_tx_ring(struct emac_instance *dev)
1059 {
1060         int i;
1061
1062         for (i = 0; i < NUM_TX_BUFF; ++i) {
1063                 if (dev->tx_skb[i]) {
1064                         dev_kfree_skb(dev->tx_skb[i]);
1065                         dev->tx_skb[i] = NULL;
1066                         if (dev->tx_desc[i].ctrl & MAL_TX_CTRL_READY)
1067                                 ++dev->estats.tx_dropped;
1068                 }
1069                 dev->tx_desc[i].ctrl = 0;
1070                 dev->tx_desc[i].data_ptr = 0;
1071         }
1072 }
1073
1074 static void emac_clean_rx_ring(struct emac_instance *dev)
1075 {
1076         int i;
1077
1078         for (i = 0; i < NUM_RX_BUFF; ++i)
1079                 if (dev->rx_skb[i]) {
1080                         dev->rx_desc[i].ctrl = 0;
1081                         dev_kfree_skb(dev->rx_skb[i]);
1082                         dev->rx_skb[i] = NULL;
1083                         dev->rx_desc[i].data_ptr = 0;
1084                 }
1085
1086         if (dev->rx_sg_skb) {
1087                 dev_kfree_skb(dev->rx_sg_skb);
1088                 dev->rx_sg_skb = NULL;
1089         }
1090 }
1091
1092 static inline int emac_alloc_rx_skb(struct emac_instance *dev, int slot,
1093                                     gfp_t flags)
1094 {
1095         struct sk_buff *skb = alloc_skb(dev->rx_skb_size, flags);
1096         if (unlikely(!skb))
1097                 return -ENOMEM;
1098
1099         dev->rx_skb[slot] = skb;
1100         dev->rx_desc[slot].data_len = 0;
1101
1102         skb_reserve(skb, EMAC_RX_SKB_HEADROOM + 2);
1103         dev->rx_desc[slot].data_ptr =
1104             dma_map_single(&dev->ofdev->dev, skb->data - 2, dev->rx_sync_size,
1105                            DMA_FROM_DEVICE) + 2;
1106         wmb();
1107         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1108             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1109
1110         return 0;
1111 }
1112
1113 static void emac_print_link_status(struct emac_instance *dev)
1114 {
1115         if (netif_carrier_ok(dev->ndev))
1116                 printk(KERN_INFO "%s: link is up, %d %s%s\n",
1117                        dev->ndev->name, dev->phy.speed,
1118                        dev->phy.duplex == DUPLEX_FULL ? "FDX" : "HDX",
1119                        dev->phy.pause ? ", pause enabled" :
1120                        dev->phy.asym_pause ? ", asymmetric pause enabled" : "");
1121         else
1122                 printk(KERN_INFO "%s: link is down\n", dev->ndev->name);
1123 }
1124
1125 /* Process ctx, rtnl_lock semaphore */
1126 static int emac_open(struct net_device *ndev)
1127 {
1128         struct emac_instance *dev = netdev_priv(ndev);
1129         int err, i;
1130
1131         DBG(dev, "open" NL);
1132
1133         /* Setup error IRQ handler */
1134         err = request_irq(dev->emac_irq, emac_irq, 0, "EMAC", dev);
1135         if (err) {
1136                 printk(KERN_ERR "%s: failed to request IRQ %d\n",
1137                        ndev->name, dev->emac_irq);
1138                 return err;
1139         }
1140
1141         /* Allocate RX ring */
1142         for (i = 0; i < NUM_RX_BUFF; ++i)
1143                 if (emac_alloc_rx_skb(dev, i, GFP_KERNEL)) {
1144                         printk(KERN_ERR "%s: failed to allocate RX ring\n",
1145                                ndev->name);
1146                         goto oom;
1147                 }
1148
1149         dev->tx_cnt = dev->tx_slot = dev->ack_slot = dev->rx_slot = 0;
1150         clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1151         dev->rx_sg_skb = NULL;
1152
1153         mutex_lock(&dev->link_lock);
1154         dev->opened = 1;
1155
1156         /* Start PHY polling now.
1157          */
1158         if (dev->phy.address >= 0) {
1159                 int link_poll_interval;
1160                 if (dev->phy.def->ops->poll_link(&dev->phy)) {
1161                         dev->phy.def->ops->read_link(&dev->phy);
1162                         emac_rx_clk_default(dev);
1163                         netif_carrier_on(dev->ndev);
1164                         link_poll_interval = PHY_POLL_LINK_ON;
1165                 } else {
1166                         emac_rx_clk_tx(dev);
1167                         netif_carrier_off(dev->ndev);
1168                         link_poll_interval = PHY_POLL_LINK_OFF;
1169                 }
1170                 dev->link_polling = 1;
1171                 wmb();
1172                 schedule_delayed_work(&dev->link_work, link_poll_interval);
1173                 emac_print_link_status(dev);
1174         } else
1175                 netif_carrier_on(dev->ndev);
1176
1177         /* Required for Pause packet support in EMAC */
1178         dev_mc_add(ndev, default_mcast_addr, sizeof(default_mcast_addr), 1);
1179
1180         emac_configure(dev);
1181         mal_poll_add(dev->mal, &dev->commac);
1182         mal_enable_tx_channel(dev->mal, dev->mal_tx_chan);
1183         mal_set_rcbs(dev->mal, dev->mal_rx_chan, emac_rx_size(ndev->mtu));
1184         mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1185         emac_tx_enable(dev);
1186         emac_rx_enable(dev);
1187         emac_netif_start(dev);
1188
1189         mutex_unlock(&dev->link_lock);
1190
1191         return 0;
1192  oom:
1193         emac_clean_rx_ring(dev);
1194         free_irq(dev->emac_irq, dev);
1195
1196         return -ENOMEM;
1197 }
1198
1199 /* BHs disabled */
1200 #if 0
1201 static int emac_link_differs(struct emac_instance *dev)
1202 {
1203         u32 r = in_be32(&dev->emacp->mr1);
1204
1205         int duplex = r & EMAC_MR1_FDE ? DUPLEX_FULL : DUPLEX_HALF;
1206         int speed, pause, asym_pause;
1207
1208         if (r & EMAC_MR1_MF_1000)
1209                 speed = SPEED_1000;
1210         else if (r & EMAC_MR1_MF_100)
1211                 speed = SPEED_100;
1212         else
1213                 speed = SPEED_10;
1214
1215         switch (r & (EMAC_MR1_EIFC | EMAC_MR1_APP)) {
1216         case (EMAC_MR1_EIFC | EMAC_MR1_APP):
1217                 pause = 1;
1218                 asym_pause = 0;
1219                 break;
1220         case EMAC_MR1_APP:
1221                 pause = 0;
1222                 asym_pause = 1;
1223                 break;
1224         default:
1225                 pause = asym_pause = 0;
1226         }
1227         return speed != dev->phy.speed || duplex != dev->phy.duplex ||
1228             pause != dev->phy.pause || asym_pause != dev->phy.asym_pause;
1229 }
1230 #endif
1231
1232 static void emac_link_timer(struct work_struct *work)
1233 {
1234         struct emac_instance *dev =
1235                 container_of(to_delayed_work(work),
1236                              struct emac_instance, link_work);
1237         int link_poll_interval;
1238
1239         mutex_lock(&dev->link_lock);
1240         DBG2(dev, "link timer" NL);
1241
1242         if (!dev->opened)
1243                 goto bail;
1244
1245         if (dev->phy.def->ops->poll_link(&dev->phy)) {
1246                 if (!netif_carrier_ok(dev->ndev)) {
1247                         emac_rx_clk_default(dev);
1248                         /* Get new link parameters */
1249                         dev->phy.def->ops->read_link(&dev->phy);
1250
1251                         netif_carrier_on(dev->ndev);
1252                         emac_netif_stop(dev);
1253                         emac_full_tx_reset(dev);
1254                         emac_netif_start(dev);
1255                         emac_print_link_status(dev);
1256                 }
1257                 link_poll_interval = PHY_POLL_LINK_ON;
1258         } else {
1259                 if (netif_carrier_ok(dev->ndev)) {
1260                         emac_rx_clk_tx(dev);
1261                         netif_carrier_off(dev->ndev);
1262                         netif_tx_disable(dev->ndev);
1263                         emac_reinitialize(dev);
1264                         emac_print_link_status(dev);
1265                 }
1266                 link_poll_interval = PHY_POLL_LINK_OFF;
1267         }
1268         schedule_delayed_work(&dev->link_work, link_poll_interval);
1269  bail:
1270         mutex_unlock(&dev->link_lock);
1271 }
1272
1273 static void emac_force_link_update(struct emac_instance *dev)
1274 {
1275         netif_carrier_off(dev->ndev);
1276         smp_rmb();
1277         if (dev->link_polling) {
1278                 cancel_rearming_delayed_work(&dev->link_work);
1279                 if (dev->link_polling)
1280                         schedule_delayed_work(&dev->link_work,  PHY_POLL_LINK_OFF);
1281         }
1282 }
1283
1284 /* Process ctx, rtnl_lock semaphore */
1285 static int emac_close(struct net_device *ndev)
1286 {
1287         struct emac_instance *dev = netdev_priv(ndev);
1288
1289         DBG(dev, "close" NL);
1290
1291         if (dev->phy.address >= 0) {
1292                 dev->link_polling = 0;
1293                 cancel_rearming_delayed_work(&dev->link_work);
1294         }
1295         mutex_lock(&dev->link_lock);
1296         emac_netif_stop(dev);
1297         dev->opened = 0;
1298         mutex_unlock(&dev->link_lock);
1299
1300         emac_rx_disable(dev);
1301         emac_tx_disable(dev);
1302         mal_disable_rx_channel(dev->mal, dev->mal_rx_chan);
1303         mal_disable_tx_channel(dev->mal, dev->mal_tx_chan);
1304         mal_poll_del(dev->mal, &dev->commac);
1305
1306         emac_clean_tx_ring(dev);
1307         emac_clean_rx_ring(dev);
1308
1309         free_irq(dev->emac_irq, dev);
1310
1311         netif_carrier_off(ndev);
1312
1313         return 0;
1314 }
1315
1316 static inline u16 emac_tx_csum(struct emac_instance *dev,
1317                                struct sk_buff *skb)
1318 {
1319         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
1320                 (skb->ip_summed == CHECKSUM_PARTIAL)) {
1321                 ++dev->stats.tx_packets_csum;
1322                 return EMAC_TX_CTRL_TAH_CSUM;
1323         }
1324         return 0;
1325 }
1326
1327 static inline int emac_xmit_finish(struct emac_instance *dev, int len)
1328 {
1329         struct emac_regs __iomem *p = dev->emacp;
1330         struct net_device *ndev = dev->ndev;
1331
1332         /* Send the packet out. If the if makes a significant perf
1333          * difference, then we can store the TMR0 value in "dev"
1334          * instead
1335          */
1336         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
1337                 out_be32(&p->tmr0, EMAC4_TMR0_XMIT);
1338         else
1339                 out_be32(&p->tmr0, EMAC_TMR0_XMIT);
1340
1341         if (unlikely(++dev->tx_cnt == NUM_TX_BUFF)) {
1342                 netif_stop_queue(ndev);
1343                 DBG2(dev, "stopped TX queue" NL);
1344         }
1345
1346         ndev->trans_start = jiffies;
1347         ++dev->stats.tx_packets;
1348         dev->stats.tx_bytes += len;
1349
1350         return NETDEV_TX_OK;
1351 }
1352
1353 /* Tx lock BH */
1354 static int emac_start_xmit(struct sk_buff *skb, struct net_device *ndev)
1355 {
1356         struct emac_instance *dev = netdev_priv(ndev);
1357         unsigned int len = skb->len;
1358         int slot;
1359
1360         u16 ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1361             MAL_TX_CTRL_LAST | emac_tx_csum(dev, skb);
1362
1363         slot = dev->tx_slot++;
1364         if (dev->tx_slot == NUM_TX_BUFF) {
1365                 dev->tx_slot = 0;
1366                 ctrl |= MAL_TX_CTRL_WRAP;
1367         }
1368
1369         DBG2(dev, "xmit(%u) %d" NL, len, slot);
1370
1371         dev->tx_skb[slot] = skb;
1372         dev->tx_desc[slot].data_ptr = dma_map_single(&dev->ofdev->dev,
1373                                                      skb->data, len,
1374                                                      DMA_TO_DEVICE);
1375         dev->tx_desc[slot].data_len = (u16) len;
1376         wmb();
1377         dev->tx_desc[slot].ctrl = ctrl;
1378
1379         return emac_xmit_finish(dev, len);
1380 }
1381
1382 static inline int emac_xmit_split(struct emac_instance *dev, int slot,
1383                                   u32 pd, int len, int last, u16 base_ctrl)
1384 {
1385         while (1) {
1386                 u16 ctrl = base_ctrl;
1387                 int chunk = min(len, MAL_MAX_TX_SIZE);
1388                 len -= chunk;
1389
1390                 slot = (slot + 1) % NUM_TX_BUFF;
1391
1392                 if (last && !len)
1393                         ctrl |= MAL_TX_CTRL_LAST;
1394                 if (slot == NUM_TX_BUFF - 1)
1395                         ctrl |= MAL_TX_CTRL_WRAP;
1396
1397                 dev->tx_skb[slot] = NULL;
1398                 dev->tx_desc[slot].data_ptr = pd;
1399                 dev->tx_desc[slot].data_len = (u16) chunk;
1400                 dev->tx_desc[slot].ctrl = ctrl;
1401                 ++dev->tx_cnt;
1402
1403                 if (!len)
1404                         break;
1405
1406                 pd += chunk;
1407         }
1408         return slot;
1409 }
1410
1411 /* Tx lock BH disabled (SG version for TAH equipped EMACs) */
1412 static int emac_start_xmit_sg(struct sk_buff *skb, struct net_device *ndev)
1413 {
1414         struct emac_instance *dev = netdev_priv(ndev);
1415         int nr_frags = skb_shinfo(skb)->nr_frags;
1416         int len = skb->len, chunk;
1417         int slot, i;
1418         u16 ctrl;
1419         u32 pd;
1420
1421         /* This is common "fast" path */
1422         if (likely(!nr_frags && len <= MAL_MAX_TX_SIZE))
1423                 return emac_start_xmit(skb, ndev);
1424
1425         len -= skb->data_len;
1426
1427         /* Note, this is only an *estimation*, we can still run out of empty
1428          * slots because of the additional fragmentation into
1429          * MAL_MAX_TX_SIZE-sized chunks
1430          */
1431         if (unlikely(dev->tx_cnt + nr_frags + mal_tx_chunks(len) > NUM_TX_BUFF))
1432                 goto stop_queue;
1433
1434         ctrl = EMAC_TX_CTRL_GFCS | EMAC_TX_CTRL_GP | MAL_TX_CTRL_READY |
1435             emac_tx_csum(dev, skb);
1436         slot = dev->tx_slot;
1437
1438         /* skb data */
1439         dev->tx_skb[slot] = NULL;
1440         chunk = min(len, MAL_MAX_TX_SIZE);
1441         dev->tx_desc[slot].data_ptr = pd =
1442             dma_map_single(&dev->ofdev->dev, skb->data, len, DMA_TO_DEVICE);
1443         dev->tx_desc[slot].data_len = (u16) chunk;
1444         len -= chunk;
1445         if (unlikely(len))
1446                 slot = emac_xmit_split(dev, slot, pd + chunk, len, !nr_frags,
1447                                        ctrl);
1448         /* skb fragments */
1449         for (i = 0; i < nr_frags; ++i) {
1450                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
1451                 len = frag->size;
1452
1453                 if (unlikely(dev->tx_cnt + mal_tx_chunks(len) >= NUM_TX_BUFF))
1454                         goto undo_frame;
1455
1456                 pd = dma_map_page(&dev->ofdev->dev, frag->page, frag->page_offset, len,
1457                                   DMA_TO_DEVICE);
1458
1459                 slot = emac_xmit_split(dev, slot, pd, len, i == nr_frags - 1,
1460                                        ctrl);
1461         }
1462
1463         DBG2(dev, "xmit_sg(%u) %d - %d" NL, skb->len, dev->tx_slot, slot);
1464
1465         /* Attach skb to the last slot so we don't release it too early */
1466         dev->tx_skb[slot] = skb;
1467
1468         /* Send the packet out */
1469         if (dev->tx_slot == NUM_TX_BUFF - 1)
1470                 ctrl |= MAL_TX_CTRL_WRAP;
1471         wmb();
1472         dev->tx_desc[dev->tx_slot].ctrl = ctrl;
1473         dev->tx_slot = (slot + 1) % NUM_TX_BUFF;
1474
1475         return emac_xmit_finish(dev, skb->len);
1476
1477  undo_frame:
1478         /* Well, too bad. Our previous estimation was overly optimistic.
1479          * Undo everything.
1480          */
1481         while (slot != dev->tx_slot) {
1482                 dev->tx_desc[slot].ctrl = 0;
1483                 --dev->tx_cnt;
1484                 if (--slot < 0)
1485                         slot = NUM_TX_BUFF - 1;
1486         }
1487         ++dev->estats.tx_undo;
1488
1489  stop_queue:
1490         netif_stop_queue(ndev);
1491         DBG2(dev, "stopped TX queue" NL);
1492         return NETDEV_TX_BUSY;
1493 }
1494
1495 /* Tx lock BHs */
1496 static void emac_parse_tx_error(struct emac_instance *dev, u16 ctrl)
1497 {
1498         struct emac_error_stats *st = &dev->estats;
1499
1500         DBG(dev, "BD TX error %04x" NL, ctrl);
1501
1502         ++st->tx_bd_errors;
1503         if (ctrl & EMAC_TX_ST_BFCS)
1504                 ++st->tx_bd_bad_fcs;
1505         if (ctrl & EMAC_TX_ST_LCS)
1506                 ++st->tx_bd_carrier_loss;
1507         if (ctrl & EMAC_TX_ST_ED)
1508                 ++st->tx_bd_excessive_deferral;
1509         if (ctrl & EMAC_TX_ST_EC)
1510                 ++st->tx_bd_excessive_collisions;
1511         if (ctrl & EMAC_TX_ST_LC)
1512                 ++st->tx_bd_late_collision;
1513         if (ctrl & EMAC_TX_ST_MC)
1514                 ++st->tx_bd_multple_collisions;
1515         if (ctrl & EMAC_TX_ST_SC)
1516                 ++st->tx_bd_single_collision;
1517         if (ctrl & EMAC_TX_ST_UR)
1518                 ++st->tx_bd_underrun;
1519         if (ctrl & EMAC_TX_ST_SQE)
1520                 ++st->tx_bd_sqe;
1521 }
1522
1523 static void emac_poll_tx(void *param)
1524 {
1525         struct emac_instance *dev = param;
1526         u32 bad_mask;
1527
1528         DBG2(dev, "poll_tx, %d %d" NL, dev->tx_cnt, dev->ack_slot);
1529
1530         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
1531                 bad_mask = EMAC_IS_BAD_TX_TAH;
1532         else
1533                 bad_mask = EMAC_IS_BAD_TX;
1534
1535         netif_tx_lock_bh(dev->ndev);
1536         if (dev->tx_cnt) {
1537                 u16 ctrl;
1538                 int slot = dev->ack_slot, n = 0;
1539         again:
1540                 ctrl = dev->tx_desc[slot].ctrl;
1541                 if (!(ctrl & MAL_TX_CTRL_READY)) {
1542                         struct sk_buff *skb = dev->tx_skb[slot];
1543                         ++n;
1544
1545                         if (skb) {
1546                                 dev_kfree_skb(skb);
1547                                 dev->tx_skb[slot] = NULL;
1548                         }
1549                         slot = (slot + 1) % NUM_TX_BUFF;
1550
1551                         if (unlikely(ctrl & bad_mask))
1552                                 emac_parse_tx_error(dev, ctrl);
1553
1554                         if (--dev->tx_cnt)
1555                                 goto again;
1556                 }
1557                 if (n) {
1558                         dev->ack_slot = slot;
1559                         if (netif_queue_stopped(dev->ndev) &&
1560                             dev->tx_cnt < EMAC_TX_WAKEUP_THRESH)
1561                                 netif_wake_queue(dev->ndev);
1562
1563                         DBG2(dev, "tx %d pkts" NL, n);
1564                 }
1565         }
1566         netif_tx_unlock_bh(dev->ndev);
1567 }
1568
1569 static inline void emac_recycle_rx_skb(struct emac_instance *dev, int slot,
1570                                        int len)
1571 {
1572         struct sk_buff *skb = dev->rx_skb[slot];
1573
1574         DBG2(dev, "recycle %d %d" NL, slot, len);
1575
1576         if (len)
1577                 dma_map_single(&dev->ofdev->dev, skb->data - 2,
1578                                EMAC_DMA_ALIGN(len + 2), DMA_FROM_DEVICE);
1579
1580         dev->rx_desc[slot].data_len = 0;
1581         wmb();
1582         dev->rx_desc[slot].ctrl = MAL_RX_CTRL_EMPTY |
1583             (slot == (NUM_RX_BUFF - 1) ? MAL_RX_CTRL_WRAP : 0);
1584 }
1585
1586 static void emac_parse_rx_error(struct emac_instance *dev, u16 ctrl)
1587 {
1588         struct emac_error_stats *st = &dev->estats;
1589
1590         DBG(dev, "BD RX error %04x" NL, ctrl);
1591
1592         ++st->rx_bd_errors;
1593         if (ctrl & EMAC_RX_ST_OE)
1594                 ++st->rx_bd_overrun;
1595         if (ctrl & EMAC_RX_ST_BP)
1596                 ++st->rx_bd_bad_packet;
1597         if (ctrl & EMAC_RX_ST_RP)
1598                 ++st->rx_bd_runt_packet;
1599         if (ctrl & EMAC_RX_ST_SE)
1600                 ++st->rx_bd_short_event;
1601         if (ctrl & EMAC_RX_ST_AE)
1602                 ++st->rx_bd_alignment_error;
1603         if (ctrl & EMAC_RX_ST_BFCS)
1604                 ++st->rx_bd_bad_fcs;
1605         if (ctrl & EMAC_RX_ST_PTL)
1606                 ++st->rx_bd_packet_too_long;
1607         if (ctrl & EMAC_RX_ST_ORE)
1608                 ++st->rx_bd_out_of_range;
1609         if (ctrl & EMAC_RX_ST_IRE)
1610                 ++st->rx_bd_in_range;
1611 }
1612
1613 static inline void emac_rx_csum(struct emac_instance *dev,
1614                                 struct sk_buff *skb, u16 ctrl)
1615 {
1616 #ifdef CONFIG_IBM_NEW_EMAC_TAH
1617         if (!ctrl && dev->tah_dev) {
1618                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1619                 ++dev->stats.rx_packets_csum;
1620         }
1621 #endif
1622 }
1623
1624 static inline int emac_rx_sg_append(struct emac_instance *dev, int slot)
1625 {
1626         if (likely(dev->rx_sg_skb != NULL)) {
1627                 int len = dev->rx_desc[slot].data_len;
1628                 int tot_len = dev->rx_sg_skb->len + len;
1629
1630                 if (unlikely(tot_len + 2 > dev->rx_skb_size)) {
1631                         ++dev->estats.rx_dropped_mtu;
1632                         dev_kfree_skb(dev->rx_sg_skb);
1633                         dev->rx_sg_skb = NULL;
1634                 } else {
1635                         cacheable_memcpy(skb_tail_pointer(dev->rx_sg_skb),
1636                                          dev->rx_skb[slot]->data, len);
1637                         skb_put(dev->rx_sg_skb, len);
1638                         emac_recycle_rx_skb(dev, slot, len);
1639                         return 0;
1640                 }
1641         }
1642         emac_recycle_rx_skb(dev, slot, 0);
1643         return -1;
1644 }
1645
1646 /* NAPI poll context */
1647 static int emac_poll_rx(void *param, int budget)
1648 {
1649         struct emac_instance *dev = param;
1650         int slot = dev->rx_slot, received = 0;
1651
1652         DBG2(dev, "poll_rx(%d)" NL, budget);
1653
1654  again:
1655         while (budget > 0) {
1656                 int len;
1657                 struct sk_buff *skb;
1658                 u16 ctrl = dev->rx_desc[slot].ctrl;
1659
1660                 if (ctrl & MAL_RX_CTRL_EMPTY)
1661                         break;
1662
1663                 skb = dev->rx_skb[slot];
1664                 mb();
1665                 len = dev->rx_desc[slot].data_len;
1666
1667                 if (unlikely(!MAL_IS_SINGLE_RX(ctrl)))
1668                         goto sg;
1669
1670                 ctrl &= EMAC_BAD_RX_MASK;
1671                 if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1672                         emac_parse_rx_error(dev, ctrl);
1673                         ++dev->estats.rx_dropped_error;
1674                         emac_recycle_rx_skb(dev, slot, 0);
1675                         len = 0;
1676                         goto next;
1677                 }
1678
1679                 if (len < ETH_HLEN) {
1680                         ++dev->estats.rx_dropped_stack;
1681                         emac_recycle_rx_skb(dev, slot, len);
1682                         goto next;
1683                 }
1684
1685                 if (len && len < EMAC_RX_COPY_THRESH) {
1686                         struct sk_buff *copy_skb =
1687                             alloc_skb(len + EMAC_RX_SKB_HEADROOM + 2, GFP_ATOMIC);
1688                         if (unlikely(!copy_skb))
1689                                 goto oom;
1690
1691                         skb_reserve(copy_skb, EMAC_RX_SKB_HEADROOM + 2);
1692                         cacheable_memcpy(copy_skb->data - 2, skb->data - 2,
1693                                          len + 2);
1694                         emac_recycle_rx_skb(dev, slot, len);
1695                         skb = copy_skb;
1696                 } else if (unlikely(emac_alloc_rx_skb(dev, slot, GFP_ATOMIC)))
1697                         goto oom;
1698
1699                 skb_put(skb, len);
1700         push_packet:
1701                 skb->dev = dev->ndev;
1702                 skb->protocol = eth_type_trans(skb, dev->ndev);
1703                 emac_rx_csum(dev, skb, ctrl);
1704
1705                 if (unlikely(netif_receive_skb(skb) == NET_RX_DROP))
1706                         ++dev->estats.rx_dropped_stack;
1707         next:
1708                 ++dev->stats.rx_packets;
1709         skip:
1710                 dev->stats.rx_bytes += len;
1711                 slot = (slot + 1) % NUM_RX_BUFF;
1712                 --budget;
1713                 ++received;
1714                 continue;
1715         sg:
1716                 if (ctrl & MAL_RX_CTRL_FIRST) {
1717                         BUG_ON(dev->rx_sg_skb);
1718                         if (unlikely(emac_alloc_rx_skb(dev, slot, GFP_ATOMIC))) {
1719                                 DBG(dev, "rx OOM %d" NL, slot);
1720                                 ++dev->estats.rx_dropped_oom;
1721                                 emac_recycle_rx_skb(dev, slot, 0);
1722                         } else {
1723                                 dev->rx_sg_skb = skb;
1724                                 skb_put(skb, len);
1725                         }
1726                 } else if (!emac_rx_sg_append(dev, slot) &&
1727                            (ctrl & MAL_RX_CTRL_LAST)) {
1728
1729                         skb = dev->rx_sg_skb;
1730                         dev->rx_sg_skb = NULL;
1731
1732                         ctrl &= EMAC_BAD_RX_MASK;
1733                         if (unlikely(ctrl && ctrl != EMAC_RX_TAH_BAD_CSUM)) {
1734                                 emac_parse_rx_error(dev, ctrl);
1735                                 ++dev->estats.rx_dropped_error;
1736                                 dev_kfree_skb(skb);
1737                                 len = 0;
1738                         } else
1739                                 goto push_packet;
1740                 }
1741                 goto skip;
1742         oom:
1743                 DBG(dev, "rx OOM %d" NL, slot);
1744                 /* Drop the packet and recycle skb */
1745                 ++dev->estats.rx_dropped_oom;
1746                 emac_recycle_rx_skb(dev, slot, 0);
1747                 goto next;
1748         }
1749
1750         if (received) {
1751                 DBG2(dev, "rx %d BDs" NL, received);
1752                 dev->rx_slot = slot;
1753         }
1754
1755         if (unlikely(budget && test_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags))) {
1756                 mb();
1757                 if (!(dev->rx_desc[slot].ctrl & MAL_RX_CTRL_EMPTY)) {
1758                         DBG2(dev, "rx restart" NL);
1759                         received = 0;
1760                         goto again;
1761                 }
1762
1763                 if (dev->rx_sg_skb) {
1764                         DBG2(dev, "dropping partial rx packet" NL);
1765                         ++dev->estats.rx_dropped_error;
1766                         dev_kfree_skb(dev->rx_sg_skb);
1767                         dev->rx_sg_skb = NULL;
1768                 }
1769
1770                 clear_bit(MAL_COMMAC_RX_STOPPED, &dev->commac.flags);
1771                 mal_enable_rx_channel(dev->mal, dev->mal_rx_chan);
1772                 emac_rx_enable(dev);
1773                 dev->rx_slot = 0;
1774         }
1775         return received;
1776 }
1777
1778 /* NAPI poll context */
1779 static int emac_peek_rx(void *param)
1780 {
1781         struct emac_instance *dev = param;
1782
1783         return !(dev->rx_desc[dev->rx_slot].ctrl & MAL_RX_CTRL_EMPTY);
1784 }
1785
1786 /* NAPI poll context */
1787 static int emac_peek_rx_sg(void *param)
1788 {
1789         struct emac_instance *dev = param;
1790
1791         int slot = dev->rx_slot;
1792         while (1) {
1793                 u16 ctrl = dev->rx_desc[slot].ctrl;
1794                 if (ctrl & MAL_RX_CTRL_EMPTY)
1795                         return 0;
1796                 else if (ctrl & MAL_RX_CTRL_LAST)
1797                         return 1;
1798
1799                 slot = (slot + 1) % NUM_RX_BUFF;
1800
1801                 /* I'm just being paranoid here :) */
1802                 if (unlikely(slot == dev->rx_slot))
1803                         return 0;
1804         }
1805 }
1806
1807 /* Hard IRQ */
1808 static void emac_rxde(void *param)
1809 {
1810         struct emac_instance *dev = param;
1811
1812         ++dev->estats.rx_stopped;
1813         emac_rx_disable_async(dev);
1814 }
1815
1816 /* Hard IRQ */
1817 static irqreturn_t emac_irq(int irq, void *dev_instance)
1818 {
1819         struct emac_instance *dev = dev_instance;
1820         struct emac_regs __iomem *p = dev->emacp;
1821         struct emac_error_stats *st = &dev->estats;
1822         u32 isr;
1823
1824         spin_lock(&dev->lock);
1825
1826         isr = in_be32(&p->isr);
1827         out_be32(&p->isr, isr);
1828
1829         DBG(dev, "isr = %08x" NL, isr);
1830
1831         if (isr & EMAC4_ISR_TXPE)
1832                 ++st->tx_parity;
1833         if (isr & EMAC4_ISR_RXPE)
1834                 ++st->rx_parity;
1835         if (isr & EMAC4_ISR_TXUE)
1836                 ++st->tx_underrun;
1837         if (isr & EMAC4_ISR_RXOE)
1838                 ++st->rx_fifo_overrun;
1839         if (isr & EMAC_ISR_OVR)
1840                 ++st->rx_overrun;
1841         if (isr & EMAC_ISR_BP)
1842                 ++st->rx_bad_packet;
1843         if (isr & EMAC_ISR_RP)
1844                 ++st->rx_runt_packet;
1845         if (isr & EMAC_ISR_SE)
1846                 ++st->rx_short_event;
1847         if (isr & EMAC_ISR_ALE)
1848                 ++st->rx_alignment_error;
1849         if (isr & EMAC_ISR_BFCS)
1850                 ++st->rx_bad_fcs;
1851         if (isr & EMAC_ISR_PTLE)
1852                 ++st->rx_packet_too_long;
1853         if (isr & EMAC_ISR_ORE)
1854                 ++st->rx_out_of_range;
1855         if (isr & EMAC_ISR_IRE)
1856                 ++st->rx_in_range;
1857         if (isr & EMAC_ISR_SQE)
1858                 ++st->tx_sqe;
1859         if (isr & EMAC_ISR_TE)
1860                 ++st->tx_errors;
1861
1862         spin_unlock(&dev->lock);
1863
1864         return IRQ_HANDLED;
1865 }
1866
1867 static struct net_device_stats *emac_stats(struct net_device *ndev)
1868 {
1869         struct emac_instance *dev = netdev_priv(ndev);
1870         struct emac_stats *st = &dev->stats;
1871         struct emac_error_stats *est = &dev->estats;
1872         struct net_device_stats *nst = &dev->nstats;
1873         unsigned long flags;
1874
1875         DBG2(dev, "stats" NL);
1876
1877         /* Compute "legacy" statistics */
1878         spin_lock_irqsave(&dev->lock, flags);
1879         nst->rx_packets = (unsigned long)st->rx_packets;
1880         nst->rx_bytes = (unsigned long)st->rx_bytes;
1881         nst->tx_packets = (unsigned long)st->tx_packets;
1882         nst->tx_bytes = (unsigned long)st->tx_bytes;
1883         nst->rx_dropped = (unsigned long)(est->rx_dropped_oom +
1884                                           est->rx_dropped_error +
1885                                           est->rx_dropped_resize +
1886                                           est->rx_dropped_mtu);
1887         nst->tx_dropped = (unsigned long)est->tx_dropped;
1888
1889         nst->rx_errors = (unsigned long)est->rx_bd_errors;
1890         nst->rx_fifo_errors = (unsigned long)(est->rx_bd_overrun +
1891                                               est->rx_fifo_overrun +
1892                                               est->rx_overrun);
1893         nst->rx_frame_errors = (unsigned long)(est->rx_bd_alignment_error +
1894                                                est->rx_alignment_error);
1895         nst->rx_crc_errors = (unsigned long)(est->rx_bd_bad_fcs +
1896                                              est->rx_bad_fcs);
1897         nst->rx_length_errors = (unsigned long)(est->rx_bd_runt_packet +
1898                                                 est->rx_bd_short_event +
1899                                                 est->rx_bd_packet_too_long +
1900                                                 est->rx_bd_out_of_range +
1901                                                 est->rx_bd_in_range +
1902                                                 est->rx_runt_packet +
1903                                                 est->rx_short_event +
1904                                                 est->rx_packet_too_long +
1905                                                 est->rx_out_of_range +
1906                                                 est->rx_in_range);
1907
1908         nst->tx_errors = (unsigned long)(est->tx_bd_errors + est->tx_errors);
1909         nst->tx_fifo_errors = (unsigned long)(est->tx_bd_underrun +
1910                                               est->tx_underrun);
1911         nst->tx_carrier_errors = (unsigned long)est->tx_bd_carrier_loss;
1912         nst->collisions = (unsigned long)(est->tx_bd_excessive_deferral +
1913                                           est->tx_bd_excessive_collisions +
1914                                           est->tx_bd_late_collision +
1915                                           est->tx_bd_multple_collisions);
1916         spin_unlock_irqrestore(&dev->lock, flags);
1917         return nst;
1918 }
1919
1920 static struct mal_commac_ops emac_commac_ops = {
1921         .poll_tx = &emac_poll_tx,
1922         .poll_rx = &emac_poll_rx,
1923         .peek_rx = &emac_peek_rx,
1924         .rxde = &emac_rxde,
1925 };
1926
1927 static struct mal_commac_ops emac_commac_sg_ops = {
1928         .poll_tx = &emac_poll_tx,
1929         .poll_rx = &emac_poll_rx,
1930         .peek_rx = &emac_peek_rx_sg,
1931         .rxde = &emac_rxde,
1932 };
1933
1934 /* Ethtool support */
1935 static int emac_ethtool_get_settings(struct net_device *ndev,
1936                                      struct ethtool_cmd *cmd)
1937 {
1938         struct emac_instance *dev = netdev_priv(ndev);
1939
1940         cmd->supported = dev->phy.features;
1941         cmd->port = PORT_MII;
1942         cmd->phy_address = dev->phy.address;
1943         cmd->transceiver =
1944             dev->phy.address >= 0 ? XCVR_EXTERNAL : XCVR_INTERNAL;
1945
1946         mutex_lock(&dev->link_lock);
1947         cmd->advertising = dev->phy.advertising;
1948         cmd->autoneg = dev->phy.autoneg;
1949         cmd->speed = dev->phy.speed;
1950         cmd->duplex = dev->phy.duplex;
1951         mutex_unlock(&dev->link_lock);
1952
1953         return 0;
1954 }
1955
1956 static int emac_ethtool_set_settings(struct net_device *ndev,
1957                                      struct ethtool_cmd *cmd)
1958 {
1959         struct emac_instance *dev = netdev_priv(ndev);
1960         u32 f = dev->phy.features;
1961
1962         DBG(dev, "set_settings(%d, %d, %d, 0x%08x)" NL,
1963             cmd->autoneg, cmd->speed, cmd->duplex, cmd->advertising);
1964
1965         /* Basic sanity checks */
1966         if (dev->phy.address < 0)
1967                 return -EOPNOTSUPP;
1968         if (cmd->autoneg != AUTONEG_ENABLE && cmd->autoneg != AUTONEG_DISABLE)
1969                 return -EINVAL;
1970         if (cmd->autoneg == AUTONEG_ENABLE && cmd->advertising == 0)
1971                 return -EINVAL;
1972         if (cmd->duplex != DUPLEX_HALF && cmd->duplex != DUPLEX_FULL)
1973                 return -EINVAL;
1974
1975         if (cmd->autoneg == AUTONEG_DISABLE) {
1976                 switch (cmd->speed) {
1977                 case SPEED_10:
1978                         if (cmd->duplex == DUPLEX_HALF
1979                             && !(f & SUPPORTED_10baseT_Half))
1980                                 return -EINVAL;
1981                         if (cmd->duplex == DUPLEX_FULL
1982                             && !(f & SUPPORTED_10baseT_Full))
1983                                 return -EINVAL;
1984                         break;
1985                 case SPEED_100:
1986                         if (cmd->duplex == DUPLEX_HALF
1987                             && !(f & SUPPORTED_100baseT_Half))
1988                                 return -EINVAL;
1989                         if (cmd->duplex == DUPLEX_FULL
1990                             && !(f & SUPPORTED_100baseT_Full))
1991                                 return -EINVAL;
1992                         break;
1993                 case SPEED_1000:
1994                         if (cmd->duplex == DUPLEX_HALF
1995                             && !(f & SUPPORTED_1000baseT_Half))
1996                                 return -EINVAL;
1997                         if (cmd->duplex == DUPLEX_FULL
1998                             && !(f & SUPPORTED_1000baseT_Full))
1999                                 return -EINVAL;
2000                         break;
2001                 default:
2002                         return -EINVAL;
2003                 }
2004
2005                 mutex_lock(&dev->link_lock);
2006                 dev->phy.def->ops->setup_forced(&dev->phy, cmd->speed,
2007                                                 cmd->duplex);
2008                 mutex_unlock(&dev->link_lock);
2009
2010         } else {
2011                 if (!(f & SUPPORTED_Autoneg))
2012                         return -EINVAL;
2013
2014                 mutex_lock(&dev->link_lock);
2015                 dev->phy.def->ops->setup_aneg(&dev->phy,
2016                                               (cmd->advertising & f) |
2017                                               (dev->phy.advertising &
2018                                                (ADVERTISED_Pause |
2019                                                 ADVERTISED_Asym_Pause)));
2020                 mutex_unlock(&dev->link_lock);
2021         }
2022         emac_force_link_update(dev);
2023
2024         return 0;
2025 }
2026
2027 static void emac_ethtool_get_ringparam(struct net_device *ndev,
2028                                        struct ethtool_ringparam *rp)
2029 {
2030         rp->rx_max_pending = rp->rx_pending = NUM_RX_BUFF;
2031         rp->tx_max_pending = rp->tx_pending = NUM_TX_BUFF;
2032 }
2033
2034 static void emac_ethtool_get_pauseparam(struct net_device *ndev,
2035                                         struct ethtool_pauseparam *pp)
2036 {
2037         struct emac_instance *dev = netdev_priv(ndev);
2038
2039         mutex_lock(&dev->link_lock);
2040         if ((dev->phy.features & SUPPORTED_Autoneg) &&
2041             (dev->phy.advertising & (ADVERTISED_Pause | ADVERTISED_Asym_Pause)))
2042                 pp->autoneg = 1;
2043
2044         if (dev->phy.duplex == DUPLEX_FULL) {
2045                 if (dev->phy.pause)
2046                         pp->rx_pause = pp->tx_pause = 1;
2047                 else if (dev->phy.asym_pause)
2048                         pp->tx_pause = 1;
2049         }
2050         mutex_unlock(&dev->link_lock);
2051 }
2052
2053 static u32 emac_ethtool_get_rx_csum(struct net_device *ndev)
2054 {
2055         struct emac_instance *dev = netdev_priv(ndev);
2056
2057         return dev->tah_dev != NULL;
2058 }
2059
2060 static int emac_get_regs_len(struct emac_instance *dev)
2061 {
2062         if (emac_has_feature(dev, EMAC_FTR_EMAC4))
2063                 return sizeof(struct emac_ethtool_regs_subhdr) +
2064                         EMAC4_ETHTOOL_REGS_SIZE(dev);
2065         else
2066                 return sizeof(struct emac_ethtool_regs_subhdr) +
2067                         EMAC_ETHTOOL_REGS_SIZE(dev);
2068 }
2069
2070 static int emac_ethtool_get_regs_len(struct net_device *ndev)
2071 {
2072         struct emac_instance *dev = netdev_priv(ndev);
2073         int size;
2074
2075         size = sizeof(struct emac_ethtool_regs_hdr) +
2076                 emac_get_regs_len(dev) + mal_get_regs_len(dev->mal);
2077         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2078                 size += zmii_get_regs_len(dev->zmii_dev);
2079         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2080                 size += rgmii_get_regs_len(dev->rgmii_dev);
2081         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2082                 size += tah_get_regs_len(dev->tah_dev);
2083
2084         return size;
2085 }
2086
2087 static void *emac_dump_regs(struct emac_instance *dev, void *buf)
2088 {
2089         struct emac_ethtool_regs_subhdr *hdr = buf;
2090
2091         hdr->index = dev->cell_index;
2092         if (emac_has_feature(dev, EMAC_FTR_EMAC4)) {
2093                 hdr->version = EMAC4_ETHTOOL_REGS_VER;
2094                 memcpy_fromio(hdr + 1, dev->emacp, EMAC4_ETHTOOL_REGS_SIZE(dev));
2095                 return ((void *)(hdr + 1) + EMAC4_ETHTOOL_REGS_SIZE(dev));
2096         } else {
2097                 hdr->version = EMAC_ETHTOOL_REGS_VER;
2098                 memcpy_fromio(hdr + 1, dev->emacp, EMAC_ETHTOOL_REGS_SIZE(dev));
2099                 return ((void *)(hdr + 1) + EMAC_ETHTOOL_REGS_SIZE(dev));
2100         }
2101 }
2102
2103 static void emac_ethtool_get_regs(struct net_device *ndev,
2104                                   struct ethtool_regs *regs, void *buf)
2105 {
2106         struct emac_instance *dev = netdev_priv(ndev);
2107         struct emac_ethtool_regs_hdr *hdr = buf;
2108
2109         hdr->components = 0;
2110         buf = hdr + 1;
2111
2112         buf = mal_dump_regs(dev->mal, buf);
2113         buf = emac_dump_regs(dev, buf);
2114         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII)) {
2115                 hdr->components |= EMAC_ETHTOOL_REGS_ZMII;
2116                 buf = zmii_dump_regs(dev->zmii_dev, buf);
2117         }
2118         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII)) {
2119                 hdr->components |= EMAC_ETHTOOL_REGS_RGMII;
2120                 buf = rgmii_dump_regs(dev->rgmii_dev, buf);
2121         }
2122         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH)) {
2123                 hdr->components |= EMAC_ETHTOOL_REGS_TAH;
2124                 buf = tah_dump_regs(dev->tah_dev, buf);
2125         }
2126 }
2127
2128 static int emac_ethtool_nway_reset(struct net_device *ndev)
2129 {
2130         struct emac_instance *dev = netdev_priv(ndev);
2131         int res = 0;
2132
2133         DBG(dev, "nway_reset" NL);
2134
2135         if (dev->phy.address < 0)
2136                 return -EOPNOTSUPP;
2137
2138         mutex_lock(&dev->link_lock);
2139         if (!dev->phy.autoneg) {
2140                 res = -EINVAL;
2141                 goto out;
2142         }
2143
2144         dev->phy.def->ops->setup_aneg(&dev->phy, dev->phy.advertising);
2145  out:
2146         mutex_unlock(&dev->link_lock);
2147         emac_force_link_update(dev);
2148         return res;
2149 }
2150
2151 static int emac_ethtool_get_sset_count(struct net_device *ndev, int stringset)
2152 {
2153         if (stringset == ETH_SS_STATS)
2154                 return EMAC_ETHTOOL_STATS_COUNT;
2155         else
2156                 return -EINVAL;
2157 }
2158
2159 static void emac_ethtool_get_strings(struct net_device *ndev, u32 stringset,
2160                                      u8 * buf)
2161 {
2162         if (stringset == ETH_SS_STATS)
2163                 memcpy(buf, &emac_stats_keys, sizeof(emac_stats_keys));
2164 }
2165
2166 static void emac_ethtool_get_ethtool_stats(struct net_device *ndev,
2167                                            struct ethtool_stats *estats,
2168                                            u64 * tmp_stats)
2169 {
2170         struct emac_instance *dev = netdev_priv(ndev);
2171
2172         memcpy(tmp_stats, &dev->stats, sizeof(dev->stats));
2173         tmp_stats += sizeof(dev->stats) / sizeof(u64);
2174         memcpy(tmp_stats, &dev->estats, sizeof(dev->estats));
2175 }
2176
2177 static void emac_ethtool_get_drvinfo(struct net_device *ndev,
2178                                      struct ethtool_drvinfo *info)
2179 {
2180         struct emac_instance *dev = netdev_priv(ndev);
2181
2182         strcpy(info->driver, "ibm_emac");
2183         strcpy(info->version, DRV_VERSION);
2184         info->fw_version[0] = '\0';
2185         sprintf(info->bus_info, "PPC 4xx EMAC-%d %s",
2186                 dev->cell_index, dev->ofdev->node->full_name);
2187         info->regdump_len = emac_ethtool_get_regs_len(ndev);
2188 }
2189
2190 static const struct ethtool_ops emac_ethtool_ops = {
2191         .get_settings = emac_ethtool_get_settings,
2192         .set_settings = emac_ethtool_set_settings,
2193         .get_drvinfo = emac_ethtool_get_drvinfo,
2194
2195         .get_regs_len = emac_ethtool_get_regs_len,
2196         .get_regs = emac_ethtool_get_regs,
2197
2198         .nway_reset = emac_ethtool_nway_reset,
2199
2200         .get_ringparam = emac_ethtool_get_ringparam,
2201         .get_pauseparam = emac_ethtool_get_pauseparam,
2202
2203         .get_rx_csum = emac_ethtool_get_rx_csum,
2204
2205         .get_strings = emac_ethtool_get_strings,
2206         .get_sset_count = emac_ethtool_get_sset_count,
2207         .get_ethtool_stats = emac_ethtool_get_ethtool_stats,
2208
2209         .get_link = ethtool_op_get_link,
2210         .get_tx_csum = ethtool_op_get_tx_csum,
2211         .get_sg = ethtool_op_get_sg,
2212 };
2213
2214 static int emac_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2215 {
2216         struct emac_instance *dev = netdev_priv(ndev);
2217         struct mii_ioctl_data *data = if_mii(rq);
2218
2219         DBG(dev, "ioctl %08x" NL, cmd);
2220
2221         if (dev->phy.address < 0)
2222                 return -EOPNOTSUPP;
2223
2224         switch (cmd) {
2225         case SIOCGMIIPHY:
2226                 data->phy_id = dev->phy.address;
2227                 /* Fall through */
2228         case SIOCGMIIREG:
2229                 data->val_out = emac_mdio_read(ndev, dev->phy.address,
2230                                                data->reg_num);
2231                 return 0;
2232
2233         case SIOCSMIIREG:
2234                 emac_mdio_write(ndev, dev->phy.address, data->reg_num,
2235                                 data->val_in);
2236                 return 0;
2237         default:
2238                 return -EOPNOTSUPP;
2239         }
2240 }
2241
2242 struct emac_depentry {
2243         u32                     phandle;
2244         struct device_node      *node;
2245         struct of_device        *ofdev;
2246         void                    *drvdata;
2247 };
2248
2249 #define EMAC_DEP_MAL_IDX        0
2250 #define EMAC_DEP_ZMII_IDX       1
2251 #define EMAC_DEP_RGMII_IDX      2
2252 #define EMAC_DEP_TAH_IDX        3
2253 #define EMAC_DEP_MDIO_IDX       4
2254 #define EMAC_DEP_PREV_IDX       5
2255 #define EMAC_DEP_COUNT          6
2256
2257 static int __devinit emac_check_deps(struct emac_instance *dev,
2258                                      struct emac_depentry *deps)
2259 {
2260         int i, there = 0;
2261         struct device_node *np;
2262
2263         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2264                 /* no dependency on that item, allright */
2265                 if (deps[i].phandle == 0) {
2266                         there++;
2267                         continue;
2268                 }
2269                 /* special case for blist as the dependency might go away */
2270                 if (i == EMAC_DEP_PREV_IDX) {
2271                         np = *(dev->blist - 1);
2272                         if (np == NULL) {
2273                                 deps[i].phandle = 0;
2274                                 there++;
2275                                 continue;
2276                         }
2277                         if (deps[i].node == NULL)
2278                                 deps[i].node = of_node_get(np);
2279                 }
2280                 if (deps[i].node == NULL)
2281                         deps[i].node = of_find_node_by_phandle(deps[i].phandle);
2282                 if (deps[i].node == NULL)
2283                         continue;
2284                 if (deps[i].ofdev == NULL)
2285                         deps[i].ofdev = of_find_device_by_node(deps[i].node);
2286                 if (deps[i].ofdev == NULL)
2287                         continue;
2288                 if (deps[i].drvdata == NULL)
2289                         deps[i].drvdata = dev_get_drvdata(&deps[i].ofdev->dev);
2290                 if (deps[i].drvdata != NULL)
2291                         there++;
2292         }
2293         return (there == EMAC_DEP_COUNT);
2294 }
2295
2296 static void emac_put_deps(struct emac_instance *dev)
2297 {
2298         if (dev->mal_dev)
2299                 of_dev_put(dev->mal_dev);
2300         if (dev->zmii_dev)
2301                 of_dev_put(dev->zmii_dev);
2302         if (dev->rgmii_dev)
2303                 of_dev_put(dev->rgmii_dev);
2304         if (dev->mdio_dev)
2305                 of_dev_put(dev->mdio_dev);
2306         if (dev->tah_dev)
2307                 of_dev_put(dev->tah_dev);
2308 }
2309
2310 static int __devinit emac_of_bus_notify(struct notifier_block *nb,
2311                                         unsigned long action, void *data)
2312 {
2313         /* We are only intereted in device addition */
2314         if (action == BUS_NOTIFY_BOUND_DRIVER)
2315                 wake_up_all(&emac_probe_wait);
2316         return 0;
2317 }
2318
2319 static struct notifier_block emac_of_bus_notifier __devinitdata = {
2320         .notifier_call = emac_of_bus_notify
2321 };
2322
2323 static int __devinit emac_wait_deps(struct emac_instance *dev)
2324 {
2325         struct emac_depentry deps[EMAC_DEP_COUNT];
2326         int i, err;
2327
2328         memset(&deps, 0, sizeof(deps));
2329
2330         deps[EMAC_DEP_MAL_IDX].phandle = dev->mal_ph;
2331         deps[EMAC_DEP_ZMII_IDX].phandle = dev->zmii_ph;
2332         deps[EMAC_DEP_RGMII_IDX].phandle = dev->rgmii_ph;
2333         if (dev->tah_ph)
2334                 deps[EMAC_DEP_TAH_IDX].phandle = dev->tah_ph;
2335         if (dev->mdio_ph)
2336                 deps[EMAC_DEP_MDIO_IDX].phandle = dev->mdio_ph;
2337         if (dev->blist && dev->blist > emac_boot_list)
2338                 deps[EMAC_DEP_PREV_IDX].phandle = 0xffffffffu;
2339         bus_register_notifier(&of_platform_bus_type, &emac_of_bus_notifier);
2340         wait_event_timeout(emac_probe_wait,
2341                            emac_check_deps(dev, deps),
2342                            EMAC_PROBE_DEP_TIMEOUT);
2343         bus_unregister_notifier(&of_platform_bus_type, &emac_of_bus_notifier);
2344         err = emac_check_deps(dev, deps) ? 0 : -ENODEV;
2345         for (i = 0; i < EMAC_DEP_COUNT; i++) {
2346                 if (deps[i].node)
2347                         of_node_put(deps[i].node);
2348                 if (err && deps[i].ofdev)
2349                         of_dev_put(deps[i].ofdev);
2350         }
2351         if (err == 0) {
2352                 dev->mal_dev = deps[EMAC_DEP_MAL_IDX].ofdev;
2353                 dev->zmii_dev = deps[EMAC_DEP_ZMII_IDX].ofdev;
2354                 dev->rgmii_dev = deps[EMAC_DEP_RGMII_IDX].ofdev;
2355                 dev->tah_dev = deps[EMAC_DEP_TAH_IDX].ofdev;
2356                 dev->mdio_dev = deps[EMAC_DEP_MDIO_IDX].ofdev;
2357         }
2358         if (deps[EMAC_DEP_PREV_IDX].ofdev)
2359                 of_dev_put(deps[EMAC_DEP_PREV_IDX].ofdev);
2360         return err;
2361 }
2362
2363 static int __devinit emac_read_uint_prop(struct device_node *np, const char *name,
2364                                          u32 *val, int fatal)
2365 {
2366         int len;
2367         const u32 *prop = of_get_property(np, name, &len);
2368         if (prop == NULL || len < sizeof(u32)) {
2369                 if (fatal)
2370                         printk(KERN_ERR "%s: missing %s property\n",
2371                                np->full_name, name);
2372                 return -ENODEV;
2373         }
2374         *val = *prop;
2375         return 0;
2376 }
2377
2378 static int __devinit emac_init_phy(struct emac_instance *dev)
2379 {
2380         struct device_node *np = dev->ofdev->node;
2381         struct net_device *ndev = dev->ndev;
2382         u32 phy_map, adv;
2383         int i;
2384
2385         dev->phy.dev = ndev;
2386         dev->phy.mode = dev->phy_mode;
2387
2388         /* PHY-less configuration.
2389          * XXX I probably should move these settings to the dev tree
2390          */
2391         if (dev->phy_address == 0xffffffff && dev->phy_map == 0xffffffff) {
2392                 emac_reset(dev);
2393
2394                 /* PHY-less configuration.
2395                  * XXX I probably should move these settings to the dev tree
2396                  */
2397                 dev->phy.address = -1;
2398                 dev->phy.features = SUPPORTED_MII;
2399                 if (emac_phy_supports_gige(dev->phy_mode))
2400                         dev->phy.features |= SUPPORTED_1000baseT_Full;
2401                 else
2402                         dev->phy.features |= SUPPORTED_100baseT_Full;
2403                 dev->phy.pause = 1;
2404
2405                 return 0;
2406         }
2407
2408         mutex_lock(&emac_phy_map_lock);
2409         phy_map = dev->phy_map | busy_phy_map;
2410
2411         DBG(dev, "PHY maps %08x %08x" NL, dev->phy_map, busy_phy_map);
2412
2413         dev->phy.mdio_read = emac_mdio_read;
2414         dev->phy.mdio_write = emac_mdio_write;
2415
2416         /* Enable internal clock source */
2417 #ifdef CONFIG_PPC_DCR_NATIVE
2418         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2419                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2420 #endif
2421         /* PHY clock workaround */
2422         emac_rx_clk_tx(dev);
2423
2424         /* Enable internal clock source on 440GX*/
2425 #ifdef CONFIG_PPC_DCR_NATIVE
2426         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2427                 dcri_clrset(SDR0, SDR0_MFR, 0, SDR0_MFR_ECS);
2428 #endif
2429         /* Configure EMAC with defaults so we can at least use MDIO
2430          * This is needed mostly for 440GX
2431          */
2432         if (emac_phy_gpcs(dev->phy.mode)) {
2433                 /* XXX
2434                  * Make GPCS PHY address equal to EMAC index.
2435                  * We probably should take into account busy_phy_map
2436                  * and/or phy_map here.
2437                  *
2438                  * Note that the busy_phy_map is currently global
2439                  * while it should probably be per-ASIC...
2440                  */
2441                 dev->phy.gpcs_address = dev->gpcs_address;
2442                 if (dev->phy.gpcs_address == 0xffffffff)
2443                         dev->phy.address = dev->cell_index;
2444         }
2445
2446         emac_configure(dev);
2447
2448         if (dev->phy_address != 0xffffffff)
2449                 phy_map = ~(1 << dev->phy_address);
2450
2451         for (i = 0; i < 0x20; phy_map >>= 1, ++i)
2452                 if (!(phy_map & 1)) {
2453                         int r;
2454                         busy_phy_map |= 1 << i;
2455
2456                         /* Quick check if there is a PHY at the address */
2457                         r = emac_mdio_read(dev->ndev, i, MII_BMCR);
2458                         if (r == 0xffff || r < 0)
2459                                 continue;
2460                         if (!emac_mii_phy_probe(&dev->phy, i))
2461                                 break;
2462                 }
2463
2464         /* Enable external clock source */
2465 #ifdef CONFIG_PPC_DCR_NATIVE
2466         if (emac_has_feature(dev, EMAC_FTR_440GX_PHY_CLK_FIX))
2467                 dcri_clrset(SDR0, SDR0_MFR, SDR0_MFR_ECS, 0);
2468 #endif
2469         mutex_unlock(&emac_phy_map_lock);
2470         if (i == 0x20) {
2471                 printk(KERN_WARNING "%s: can't find PHY!\n", np->full_name);
2472                 return -ENXIO;
2473         }
2474
2475         /* Init PHY */
2476         if (dev->phy.def->ops->init)
2477                 dev->phy.def->ops->init(&dev->phy);
2478
2479         /* Disable any PHY features not supported by the platform */
2480         dev->phy.def->features &= ~dev->phy_feat_exc;
2481
2482         /* Setup initial link parameters */
2483         if (dev->phy.features & SUPPORTED_Autoneg) {
2484                 adv = dev->phy.features;
2485                 if (!emac_has_feature(dev, EMAC_FTR_NO_FLOW_CONTROL_40x))
2486                         adv |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
2487                 /* Restart autonegotiation */
2488                 dev->phy.def->ops->setup_aneg(&dev->phy, adv);
2489         } else {
2490                 u32 f = dev->phy.def->features;
2491                 int speed = SPEED_10, fd = DUPLEX_HALF;
2492
2493                 /* Select highest supported speed/duplex */
2494                 if (f & SUPPORTED_1000baseT_Full) {
2495                         speed = SPEED_1000;
2496                         fd = DUPLEX_FULL;
2497                 } else if (f & SUPPORTED_1000baseT_Half)
2498                         speed = SPEED_1000;
2499                 else if (f & SUPPORTED_100baseT_Full) {
2500                         speed = SPEED_100;
2501                         fd = DUPLEX_FULL;
2502                 } else if (f & SUPPORTED_100baseT_Half)
2503                         speed = SPEED_100;
2504                 else if (f & SUPPORTED_10baseT_Full)
2505                         fd = DUPLEX_FULL;
2506
2507                 /* Force link parameters */
2508                 dev->phy.def->ops->setup_forced(&dev->phy, speed, fd);
2509         }
2510         return 0;
2511 }
2512
2513 static int __devinit emac_init_config(struct emac_instance *dev)
2514 {
2515         struct device_node *np = dev->ofdev->node;
2516         const void *p;
2517         unsigned int plen;
2518         const char *pm, *phy_modes[] = {
2519                 [PHY_MODE_NA] = "",
2520                 [PHY_MODE_MII] = "mii",
2521                 [PHY_MODE_RMII] = "rmii",
2522                 [PHY_MODE_SMII] = "smii",
2523                 [PHY_MODE_RGMII] = "rgmii",
2524                 [PHY_MODE_TBI] = "tbi",
2525                 [PHY_MODE_GMII] = "gmii",
2526                 [PHY_MODE_RTBI] = "rtbi",
2527                 [PHY_MODE_SGMII] = "sgmii",
2528         };
2529
2530         /* Read config from device-tree */
2531         if (emac_read_uint_prop(np, "mal-device", &dev->mal_ph, 1))
2532                 return -ENXIO;
2533         if (emac_read_uint_prop(np, "mal-tx-channel", &dev->mal_tx_chan, 1))
2534                 return -ENXIO;
2535         if (emac_read_uint_prop(np, "mal-rx-channel", &dev->mal_rx_chan, 1))
2536                 return -ENXIO;
2537         if (emac_read_uint_prop(np, "cell-index", &dev->cell_index, 1))
2538                 return -ENXIO;
2539         if (emac_read_uint_prop(np, "max-frame-size", &dev->max_mtu, 0))
2540                 dev->max_mtu = 1500;
2541         if (emac_read_uint_prop(np, "rx-fifo-size", &dev->rx_fifo_size, 0))
2542                 dev->rx_fifo_size = 2048;
2543         if (emac_read_uint_prop(np, "tx-fifo-size", &dev->tx_fifo_size, 0))
2544                 dev->tx_fifo_size = 2048;
2545         if (emac_read_uint_prop(np, "rx-fifo-size-gige", &dev->rx_fifo_size_gige, 0))
2546                 dev->rx_fifo_size_gige = dev->rx_fifo_size;
2547         if (emac_read_uint_prop(np, "tx-fifo-size-gige", &dev->tx_fifo_size_gige, 0))
2548                 dev->tx_fifo_size_gige = dev->tx_fifo_size;
2549         if (emac_read_uint_prop(np, "phy-address", &dev->phy_address, 0))
2550                 dev->phy_address = 0xffffffff;
2551         if (emac_read_uint_prop(np, "phy-map", &dev->phy_map, 0))
2552                 dev->phy_map = 0xffffffff;
2553         if (emac_read_uint_prop(np, "gpcs-address", &dev->gpcs_address, 0))
2554                 dev->gpcs_address = 0xffffffff;
2555         if (emac_read_uint_prop(np->parent, "clock-frequency", &dev->opb_bus_freq, 1))
2556                 return -ENXIO;
2557         if (emac_read_uint_prop(np, "tah-device", &dev->tah_ph, 0))
2558                 dev->tah_ph = 0;
2559         if (emac_read_uint_prop(np, "tah-channel", &dev->tah_port, 0))
2560                 dev->tah_port = 0;
2561         if (emac_read_uint_prop(np, "mdio-device", &dev->mdio_ph, 0))
2562                 dev->mdio_ph = 0;
2563         if (emac_read_uint_prop(np, "zmii-device", &dev->zmii_ph, 0))
2564                 dev->zmii_ph = 0;
2565         if (emac_read_uint_prop(np, "zmii-channel", &dev->zmii_port, 0))
2566                 dev->zmii_port = 0xffffffff;
2567         if (emac_read_uint_prop(np, "rgmii-device", &dev->rgmii_ph, 0))
2568                 dev->rgmii_ph = 0;
2569         if (emac_read_uint_prop(np, "rgmii-channel", &dev->rgmii_port, 0))
2570                 dev->rgmii_port = 0xffffffff;
2571         if (emac_read_uint_prop(np, "fifo-entry-size", &dev->fifo_entry_size, 0))
2572                 dev->fifo_entry_size = 16;
2573         if (emac_read_uint_prop(np, "mal-burst-size", &dev->mal_burst_size, 0))
2574                 dev->mal_burst_size = 256;
2575
2576         /* PHY mode needs some decoding */
2577         dev->phy_mode = PHY_MODE_NA;
2578         pm = of_get_property(np, "phy-mode", &plen);
2579         if (pm != NULL) {
2580                 int i;
2581                 for (i = 0; i < ARRAY_SIZE(phy_modes); i++)
2582                         if (!strcasecmp(pm, phy_modes[i])) {
2583                                 dev->phy_mode = i;
2584                                 break;
2585                         }
2586         }
2587
2588         /* Backward compat with non-final DT */
2589         if (dev->phy_mode == PHY_MODE_NA && pm != NULL && plen == 4) {
2590                 u32 nmode = *(const u32 *)pm;
2591                 if (nmode > PHY_MODE_NA && nmode <= PHY_MODE_SGMII)
2592                         dev->phy_mode = nmode;
2593         }
2594
2595         /* Check EMAC version */
2596         if (of_device_is_compatible(np, "ibm,emac4sync")) {
2597                 dev->features |= (EMAC_FTR_EMAC4 | EMAC_FTR_EMAC4SYNC);
2598                 if (of_device_is_compatible(np, "ibm,emac-460ex") ||
2599                     of_device_is_compatible(np, "ibm,emac-460gt"))
2600                         dev->features |= EMAC_FTR_460EX_PHY_CLK_FIX;
2601                 if (of_device_is_compatible(np, "ibm,emac-405ex") ||
2602                     of_device_is_compatible(np, "ibm,emac-405exr"))
2603                         dev->features |= EMAC_FTR_440EP_PHY_CLK_FIX;
2604         } else if (of_device_is_compatible(np, "ibm,emac4")) {
2605                 dev->features |= EMAC_FTR_EMAC4;
2606                 if (of_device_is_compatible(np, "ibm,emac-440gx"))
2607                         dev->features |= EMAC_FTR_440GX_PHY_CLK_FIX;
2608         } else {
2609                 if (of_device_is_compatible(np, "ibm,emac-440ep") ||
2610                     of_device_is_compatible(np, "ibm,emac-440gr"))
2611                         dev->features |= EMAC_FTR_440EP_PHY_CLK_FIX;
2612                 if (of_device_is_compatible(np, "ibm,emac-405ez")) {
2613 #ifdef CONFIG_IBM_NEW_EMAC_NO_FLOW_CTRL
2614                         dev->features |= EMAC_FTR_NO_FLOW_CONTROL_40x;
2615 #else
2616                         printk(KERN_ERR "%s: Flow control not disabled!\n",
2617                                         np->full_name);
2618                         return -ENXIO;
2619 #endif
2620                 }
2621
2622         }
2623
2624         /* Fixup some feature bits based on the device tree */
2625         if (of_get_property(np, "has-inverted-stacr-oc", NULL))
2626                 dev->features |= EMAC_FTR_STACR_OC_INVERT;
2627         if (of_get_property(np, "has-new-stacr-staopc", NULL))
2628                 dev->features |= EMAC_FTR_HAS_NEW_STACR;
2629
2630         /* CAB lacks the appropriate properties */
2631         if (of_device_is_compatible(np, "ibm,emac-axon"))
2632                 dev->features |= EMAC_FTR_HAS_NEW_STACR |
2633                         EMAC_FTR_STACR_OC_INVERT;
2634
2635         /* Enable TAH/ZMII/RGMII features as found */
2636         if (dev->tah_ph != 0) {
2637 #ifdef CONFIG_IBM_NEW_EMAC_TAH
2638                 dev->features |= EMAC_FTR_HAS_TAH;
2639 #else
2640                 printk(KERN_ERR "%s: TAH support not enabled !\n",
2641                        np->full_name);
2642                 return -ENXIO;
2643 #endif
2644         }
2645
2646         if (dev->zmii_ph != 0) {
2647 #ifdef CONFIG_IBM_NEW_EMAC_ZMII
2648                 dev->features |= EMAC_FTR_HAS_ZMII;
2649 #else
2650                 printk(KERN_ERR "%s: ZMII support not enabled !\n",
2651                        np->full_name);
2652                 return -ENXIO;
2653 #endif
2654         }
2655
2656         if (dev->rgmii_ph != 0) {
2657 #ifdef CONFIG_IBM_NEW_EMAC_RGMII
2658                 dev->features |= EMAC_FTR_HAS_RGMII;
2659 #else
2660                 printk(KERN_ERR "%s: RGMII support not enabled !\n",
2661                        np->full_name);
2662                 return -ENXIO;
2663 #endif
2664         }
2665
2666         /* Read MAC-address */
2667         p = of_get_property(np, "local-mac-address", NULL);
2668         if (p == NULL) {
2669                 printk(KERN_ERR "%s: Can't find local-mac-address property\n",
2670                        np->full_name);
2671                 return -ENXIO;
2672         }
2673         memcpy(dev->ndev->dev_addr, p, 6);
2674
2675         /* IAHT and GAHT filter parameterization */
2676         if (emac_has_feature(dev, EMAC_FTR_EMAC4SYNC)) {
2677                 dev->xaht_slots_shift = EMAC4SYNC_XAHT_SLOTS_SHIFT;
2678                 dev->xaht_width_shift = EMAC4SYNC_XAHT_WIDTH_SHIFT;
2679         } else {
2680                 dev->xaht_slots_shift = EMAC4_XAHT_SLOTS_SHIFT;
2681                 dev->xaht_width_shift = EMAC4_XAHT_WIDTH_SHIFT;
2682         }
2683
2684         DBG(dev, "features     : 0x%08x / 0x%08x\n", dev->features, EMAC_FTRS_POSSIBLE);
2685         DBG(dev, "tx_fifo_size : %d (%d gige)\n", dev->tx_fifo_size, dev->tx_fifo_size_gige);
2686         DBG(dev, "rx_fifo_size : %d (%d gige)\n", dev->rx_fifo_size, dev->rx_fifo_size_gige);
2687         DBG(dev, "max_mtu      : %d\n", dev->max_mtu);
2688         DBG(dev, "OPB freq     : %d\n", dev->opb_bus_freq);
2689
2690         return 0;
2691 }
2692
2693 static const struct net_device_ops emac_netdev_ops = {
2694         .ndo_open               = emac_open,
2695         .ndo_stop               = emac_close,
2696         .ndo_get_stats          = emac_stats,
2697         .ndo_set_multicast_list = emac_set_multicast_list,
2698         .ndo_do_ioctl           = emac_ioctl,
2699         .ndo_tx_timeout         = emac_tx_timeout,
2700         .ndo_validate_addr      = eth_validate_addr,
2701         .ndo_set_mac_address    = eth_mac_addr,
2702         .ndo_start_xmit         = emac_start_xmit,
2703         .ndo_change_mtu         = eth_change_mtu,
2704 };
2705
2706 static const struct net_device_ops emac_gige_netdev_ops = {
2707         .ndo_open               = emac_open,
2708         .ndo_stop               = emac_close,
2709         .ndo_get_stats          = emac_stats,
2710         .ndo_set_multicast_list = emac_set_multicast_list,
2711         .ndo_do_ioctl           = emac_ioctl,
2712         .ndo_tx_timeout         = emac_tx_timeout,
2713         .ndo_validate_addr      = eth_validate_addr,
2714         .ndo_set_mac_address    = eth_mac_addr,
2715         .ndo_start_xmit         = emac_start_xmit_sg,
2716         .ndo_change_mtu         = emac_change_mtu,
2717 };
2718
2719 static int __devinit emac_probe(struct of_device *ofdev,
2720                                 const struct of_device_id *match)
2721 {
2722         struct net_device *ndev;
2723         struct emac_instance *dev;
2724         struct device_node *np = ofdev->node;
2725         struct device_node **blist = NULL;
2726         int err, i;
2727
2728         /* Skip unused/unwired EMACS.  We leave the check for an unused
2729          * property here for now, but new flat device trees should set a
2730          * status property to "disabled" instead.
2731          */
2732         if (of_get_property(np, "unused", NULL) || !of_device_is_available(np))
2733                 return -ENODEV;
2734
2735         /* Find ourselves in the bootlist if we are there */
2736         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
2737                 if (emac_boot_list[i] == np)
2738                         blist = &emac_boot_list[i];
2739
2740         /* Allocate our net_device structure */
2741         err = -ENOMEM;
2742         ndev = alloc_etherdev(sizeof(struct emac_instance));
2743         if (!ndev) {
2744                 printk(KERN_ERR "%s: could not allocate ethernet device!\n",
2745                        np->full_name);
2746                 goto err_gone;
2747         }
2748         dev = netdev_priv(ndev);
2749         dev->ndev = ndev;
2750         dev->ofdev = ofdev;
2751         dev->blist = blist;
2752         SET_NETDEV_DEV(ndev, &ofdev->dev);
2753
2754         /* Initialize some embedded data structures */
2755         mutex_init(&dev->mdio_lock);
2756         mutex_init(&dev->link_lock);
2757         spin_lock_init(&dev->lock);
2758         INIT_WORK(&dev->reset_work, emac_reset_work);
2759
2760         /* Init various config data based on device-tree */
2761         err = emac_init_config(dev);
2762         if (err != 0)
2763                 goto err_free;
2764
2765         /* Get interrupts. EMAC irq is mandatory, WOL irq is optional */
2766         dev->emac_irq = irq_of_parse_and_map(np, 0);
2767         dev->wol_irq = irq_of_parse_and_map(np, 1);
2768         if (dev->emac_irq == NO_IRQ) {
2769                 printk(KERN_ERR "%s: Can't map main interrupt\n", np->full_name);
2770                 goto err_free;
2771         }
2772         ndev->irq = dev->emac_irq;
2773
2774         /* Map EMAC regs */
2775         if (of_address_to_resource(np, 0, &dev->rsrc_regs)) {
2776                 printk(KERN_ERR "%s: Can't get registers address\n",
2777                        np->full_name);
2778                 goto err_irq_unmap;
2779         }
2780         // TODO : request_mem_region
2781         dev->emacp = ioremap(dev->rsrc_regs.start,
2782                              dev->rsrc_regs.end - dev->rsrc_regs.start + 1);
2783         if (dev->emacp == NULL) {
2784                 printk(KERN_ERR "%s: Can't map device registers!\n",
2785                        np->full_name);
2786                 err = -ENOMEM;
2787                 goto err_irq_unmap;
2788         }
2789
2790         /* Wait for dependent devices */
2791         err = emac_wait_deps(dev);
2792         if (err) {
2793                 printk(KERN_ERR
2794                        "%s: Timeout waiting for dependent devices\n",
2795                        np->full_name);
2796                 /*  display more info about what's missing ? */
2797                 goto err_reg_unmap;
2798         }
2799         dev->mal = dev_get_drvdata(&dev->mal_dev->dev);
2800         if (dev->mdio_dev != NULL)
2801                 dev->mdio_instance = dev_get_drvdata(&dev->mdio_dev->dev);
2802
2803         /* Register with MAL */
2804         dev->commac.ops = &emac_commac_ops;
2805         dev->commac.dev = dev;
2806         dev->commac.tx_chan_mask = MAL_CHAN_MASK(dev->mal_tx_chan);
2807         dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan);
2808         err = mal_register_commac(dev->mal, &dev->commac);
2809         if (err) {
2810                 printk(KERN_ERR "%s: failed to register with mal %s!\n",
2811                        np->full_name, dev->mal_dev->node->full_name);
2812                 goto err_rel_deps;
2813         }
2814         dev->rx_skb_size = emac_rx_skb_size(ndev->mtu);
2815         dev->rx_sync_size = emac_rx_sync_size(ndev->mtu);
2816
2817         /* Get pointers to BD rings */
2818         dev->tx_desc =
2819             dev->mal->bd_virt + mal_tx_bd_offset(dev->mal, dev->mal_tx_chan);
2820         dev->rx_desc =
2821             dev->mal->bd_virt + mal_rx_bd_offset(dev->mal, dev->mal_rx_chan);
2822
2823         DBG(dev, "tx_desc %p" NL, dev->tx_desc);
2824         DBG(dev, "rx_desc %p" NL, dev->rx_desc);
2825
2826         /* Clean rings */
2827         memset(dev->tx_desc, 0, NUM_TX_BUFF * sizeof(struct mal_descriptor));
2828         memset(dev->rx_desc, 0, NUM_RX_BUFF * sizeof(struct mal_descriptor));
2829         memset(dev->tx_skb, 0, NUM_TX_BUFF * sizeof(struct sk_buff *));
2830         memset(dev->rx_skb, 0, NUM_RX_BUFF * sizeof(struct sk_buff *));
2831
2832         /* Attach to ZMII, if needed */
2833         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII) &&
2834             (err = zmii_attach(dev->zmii_dev, dev->zmii_port, &dev->phy_mode)) != 0)
2835                 goto err_unreg_commac;
2836
2837         /* Attach to RGMII, if needed */
2838         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII) &&
2839             (err = rgmii_attach(dev->rgmii_dev, dev->rgmii_port, dev->phy_mode)) != 0)
2840                 goto err_detach_zmii;
2841
2842         /* Attach to TAH, if needed */
2843         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH) &&
2844             (err = tah_attach(dev->tah_dev, dev->tah_port)) != 0)
2845                 goto err_detach_rgmii;
2846
2847         /* Set some link defaults before we can find out real parameters */
2848         dev->phy.speed = SPEED_100;
2849         dev->phy.duplex = DUPLEX_FULL;
2850         dev->phy.autoneg = AUTONEG_DISABLE;
2851         dev->phy.pause = dev->phy.asym_pause = 0;
2852         dev->stop_timeout = STOP_TIMEOUT_100;
2853         INIT_DELAYED_WORK(&dev->link_work, emac_link_timer);
2854
2855         /* Find PHY if any */
2856         err = emac_init_phy(dev);
2857         if (err != 0)
2858                 goto err_detach_tah;
2859
2860         if (dev->tah_dev)
2861                 ndev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
2862         ndev->watchdog_timeo = 5 * HZ;
2863         if (emac_phy_supports_gige(dev->phy_mode)) {
2864                 ndev->netdev_ops = &emac_gige_netdev_ops;
2865                 dev->commac.ops = &emac_commac_sg_ops;
2866         } else
2867                 ndev->netdev_ops = &emac_netdev_ops;
2868         SET_ETHTOOL_OPS(ndev, &emac_ethtool_ops);
2869
2870         netif_carrier_off(ndev);
2871         netif_stop_queue(ndev);
2872
2873         err = register_netdev(ndev);
2874         if (err) {
2875                 printk(KERN_ERR "%s: failed to register net device (%d)!\n",
2876                        np->full_name, err);
2877                 goto err_detach_tah;
2878         }
2879
2880         /* Set our drvdata last as we don't want them visible until we are
2881          * fully initialized
2882          */
2883         wmb();
2884         dev_set_drvdata(&ofdev->dev, dev);
2885
2886         /* There's a new kid in town ! Let's tell everybody */
2887         wake_up_all(&emac_probe_wait);
2888
2889
2890         printk(KERN_INFO "%s: EMAC-%d %s, MAC %pM\n",
2891                ndev->name, dev->cell_index, np->full_name, ndev->dev_addr);
2892
2893         if (dev->phy_mode == PHY_MODE_SGMII)
2894                 printk(KERN_NOTICE "%s: in SGMII mode\n", ndev->name);
2895
2896         if (dev->phy.address >= 0)
2897                 printk("%s: found %s PHY (0x%02x)\n", ndev->name,
2898                        dev->phy.def->name, dev->phy.address);
2899
2900         emac_dbg_register(dev);
2901
2902         /* Life is good */
2903         return 0;
2904
2905         /* I have a bad feeling about this ... */
2906
2907  err_detach_tah:
2908         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2909                 tah_detach(dev->tah_dev, dev->tah_port);
2910  err_detach_rgmii:
2911         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2912                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
2913  err_detach_zmii:
2914         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2915                 zmii_detach(dev->zmii_dev, dev->zmii_port);
2916  err_unreg_commac:
2917         mal_unregister_commac(dev->mal, &dev->commac);
2918  err_rel_deps:
2919         emac_put_deps(dev);
2920  err_reg_unmap:
2921         iounmap(dev->emacp);
2922  err_irq_unmap:
2923         if (dev->wol_irq != NO_IRQ)
2924                 irq_dispose_mapping(dev->wol_irq);
2925         if (dev->emac_irq != NO_IRQ)
2926                 irq_dispose_mapping(dev->emac_irq);
2927  err_free:
2928         kfree(ndev);
2929  err_gone:
2930         /* if we were on the bootlist, remove us as we won't show up and
2931          * wake up all waiters to notify them in case they were waiting
2932          * on us
2933          */
2934         if (blist) {
2935                 *blist = NULL;
2936                 wake_up_all(&emac_probe_wait);
2937         }
2938         return err;
2939 }
2940
2941 static int __devexit emac_remove(struct of_device *ofdev)
2942 {
2943         struct emac_instance *dev = dev_get_drvdata(&ofdev->dev);
2944
2945         DBG(dev, "remove" NL);
2946
2947         dev_set_drvdata(&ofdev->dev, NULL);
2948
2949         unregister_netdev(dev->ndev);
2950
2951         flush_scheduled_work();
2952
2953         if (emac_has_feature(dev, EMAC_FTR_HAS_TAH))
2954                 tah_detach(dev->tah_dev, dev->tah_port);
2955         if (emac_has_feature(dev, EMAC_FTR_HAS_RGMII))
2956                 rgmii_detach(dev->rgmii_dev, dev->rgmii_port);
2957         if (emac_has_feature(dev, EMAC_FTR_HAS_ZMII))
2958                 zmii_detach(dev->zmii_dev, dev->zmii_port);
2959
2960         mal_unregister_commac(dev->mal, &dev->commac);
2961         emac_put_deps(dev);
2962
2963         emac_dbg_unregister(dev);
2964         iounmap(dev->emacp);
2965
2966         if (dev->wol_irq != NO_IRQ)
2967                 irq_dispose_mapping(dev->wol_irq);
2968         if (dev->emac_irq != NO_IRQ)
2969                 irq_dispose_mapping(dev->emac_irq);
2970
2971         kfree(dev->ndev);
2972
2973         return 0;
2974 }
2975
2976 /* XXX Features in here should be replaced by properties... */
2977 static struct of_device_id emac_match[] =
2978 {
2979         {
2980                 .type           = "network",
2981                 .compatible     = "ibm,emac",
2982         },
2983         {
2984                 .type           = "network",
2985                 .compatible     = "ibm,emac4",
2986         },
2987         {
2988                 .type           = "network",
2989                 .compatible     = "ibm,emac4sync",
2990         },
2991         {},
2992 };
2993
2994 static struct of_platform_driver emac_driver = {
2995         .name = "emac",
2996         .match_table = emac_match,
2997
2998         .probe = emac_probe,
2999         .remove = emac_remove,
3000 };
3001
3002 static void __init emac_make_bootlist(void)
3003 {
3004         struct device_node *np = NULL;
3005         int j, max, i = 0, k;
3006         int cell_indices[EMAC_BOOT_LIST_SIZE];
3007
3008         /* Collect EMACs */
3009         while((np = of_find_all_nodes(np)) != NULL) {
3010                 const u32 *idx;
3011
3012                 if (of_match_node(emac_match, np) == NULL)
3013                         continue;
3014                 if (of_get_property(np, "unused", NULL))
3015                         continue;
3016                 idx = of_get_property(np, "cell-index", NULL);
3017                 if (idx == NULL)
3018                         continue;
3019                 cell_indices[i] = *idx;
3020                 emac_boot_list[i++] = of_node_get(np);
3021                 if (i >= EMAC_BOOT_LIST_SIZE) {
3022                         of_node_put(np);
3023                         break;
3024                 }
3025         }
3026         max = i;
3027
3028         /* Bubble sort them (doh, what a creative algorithm :-) */
3029         for (i = 0; max > 1 && (i < (max - 1)); i++)
3030                 for (j = i; j < max; j++) {
3031                         if (cell_indices[i] > cell_indices[j]) {
3032                                 np = emac_boot_list[i];
3033                                 emac_boot_list[i] = emac_boot_list[j];
3034                                 emac_boot_list[j] = np;
3035                                 k = cell_indices[i];
3036                                 cell_indices[i] = cell_indices[j];
3037                                 cell_indices[j] = k;
3038                         }
3039                 }
3040 }
3041
3042 static int __init emac_init(void)
3043 {
3044         int rc;
3045
3046         printk(KERN_INFO DRV_DESC ", version " DRV_VERSION "\n");
3047
3048         /* Init debug stuff */
3049         emac_init_debug();
3050
3051         /* Build EMAC boot list */
3052         emac_make_bootlist();
3053
3054         /* Init submodules */
3055         rc = mal_init();
3056         if (rc)
3057                 goto err;
3058         rc = zmii_init();
3059         if (rc)
3060                 goto err_mal;
3061         rc = rgmii_init();
3062         if (rc)
3063                 goto err_zmii;
3064         rc = tah_init();
3065         if (rc)
3066                 goto err_rgmii;
3067         rc = of_register_platform_driver(&emac_driver);
3068         if (rc)
3069                 goto err_tah;
3070
3071         return 0;
3072
3073  err_tah:
3074         tah_exit();
3075  err_rgmii:
3076         rgmii_exit();
3077  err_zmii:
3078         zmii_exit();
3079  err_mal:
3080         mal_exit();
3081  err:
3082         return rc;
3083 }
3084
3085 static void __exit emac_exit(void)
3086 {
3087         int i;
3088
3089         of_unregister_platform_driver(&emac_driver);
3090
3091         tah_exit();
3092         rgmii_exit();
3093         zmii_exit();
3094         mal_exit();
3095         emac_fini_debug();
3096
3097         /* Destroy EMAC boot list */
3098         for (i = 0; i < EMAC_BOOT_LIST_SIZE; i++)
3099                 if (emac_boot_list[i])
3100                         of_node_put(emac_boot_list[i]);
3101 }
3102
3103 module_init(emac_init);
3104 module_exit(emac_exit);