drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005-2009 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/kernel.h>
  22 #include <linux/types.h>
  23 #include <linux/compiler.h>
  24 #include <linux/slab.h>
  25 #include <linux/delay.h>
  26 #include <linux/in.h>
  27 #include <linux/init.h>
  28 #include <linux/ioport.h>
  29 #include <linux/pci.h>
  30 #include <linux/netdevice.h>
  31 #include <linux/etherdevice.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/mii.h>
  35 #include <linux/phy.h>
  36 #include <linux/brcmphy.h>
  37 #include <linux/if_vlan.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/workqueue.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/dma-mapping.h>
  43 #include <linux/firmware.h>
  44
  45 #include <net/checksum.h>
  46 #include <net/ip.h>
  47
  48 #include <asm/system.h>
  49 #include <asm/io.h>
  50 #include <asm/byteorder.h>
  51 #include <asm/uaccess.h>
  52
  53 #ifdef CONFIG_SPARC
  54 #include <asm/idprom.h>
  55 #include <asm/prom.h>
  56 #endif
  57
  58 #define BAR_0   0
  59 #define BAR_2   2
  60
  61 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
  62 #define TG3_VLAN_TAG_USED 1
  63 #else
  64 #define TG3_VLAN_TAG_USED 0
  65 #endif
  66
  67 #include "tg3.h"
  68
  69 #define DRV_MODULE_NAME         "tg3"
  70 #define PFX DRV_MODULE_NAME     ": "
  71 #define DRV_MODULE_VERSION      "3.102"
  72 #define DRV_MODULE_RELDATE      "September 1, 2009"
  73
  74 #define TG3_DEF_MAC_MODE        0
  75 #define TG3_DEF_RX_MODE         0
  76 #define TG3_DEF_TX_MODE         0
  77 #define TG3_DEF_MSG_ENABLE        \
  78         (NETIF_MSG_DRV          | \
  79          NETIF_MSG_PROBE        | \
  80          NETIF_MSG_LINK         | \
  81          NETIF_MSG_TIMER        | \
  82          NETIF_MSG_IFDOWN       | \
  83          NETIF_MSG_IFUP         | \
  84          NETIF_MSG_RX_ERR       | \
  85          NETIF_MSG_TX_ERR)
  86
  87 /* length of time before we decide the hardware is borked,
  88  * and dev->tx_timeout() should be called to fix the problem
  89  */
  90 #define TG3_TX_TIMEOUT                  (5 * HZ)
  91
  92 /* hardware minimum and maximum for a single frame's data payload */
  93 #define TG3_MIN_MTU                     60
  94 #define TG3_MAX_MTU(tp) \
  95         ((tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) ? 9000 : 1500)
  96
  97 /* These numbers seem to be hard coded in the NIC firmware somehow.
  98  * You can't change the ring sizes, but you can change where you place
  99  * them in the NIC onboard memory.
 100  */
 101 #define TG3_RX_RING_SIZE                512
 102 #define TG3_DEF_RX_RING_PENDING         200
 103 #define TG3_RX_JUMBO_RING_SIZE          256
 104 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 105 #define TG3_RSS_INDIR_TBL_SIZE 128
 106
 107 /* Do not place this n-ring entries value into the tp struct itself,
 108  * we really want to expose these constants to GCC so that modulo et
 109  * al.  operations are done with shifts and masks instead of with
 110  * hw multiply/modulo instructions.  Another solution would be to
 111  * replace things like '% foo' with '& (foo - 1)'.
 112  */
 113 #define TG3_RX_RCB_RING_SIZE(tp)        \
 114         (((tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) && \
 115           !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) ? 1024 : 512)
 116
 117 #define TG3_TX_RING_SIZE                512
 118 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 119
 120 #define TG3_RX_RING_BYTES       (sizeof(struct tg3_rx_buffer_desc) * \
 121                                  TG3_RX_RING_SIZE)
 122 #define TG3_RX_JUMBO_RING_BYTES (sizeof(struct tg3_ext_rx_buffer_desc) * \
 123                                  TG3_RX_JUMBO_RING_SIZE)
 124 #define TG3_RX_RCB_RING_BYTES(tp) (sizeof(struct tg3_rx_buffer_desc) * \
 125                                  TG3_RX_RCB_RING_SIZE(tp))
 126 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 127                                  TG3_TX_RING_SIZE)
 128 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 129
 130 #define TG3_DMA_BYTE_ENAB               64
 131
 132 #define TG3_RX_STD_DMA_SZ               1536
 133 #define TG3_RX_JMB_DMA_SZ               9046
 134
 135 #define TG3_RX_DMA_TO_MAP_SZ(x)         ((x) + TG3_DMA_BYTE_ENAB)
 136
 137 #define TG3_RX_STD_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ)
 138 #define TG3_RX_JMB_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ)
 139
 140 /* minimum number of free TX descriptors required to wake up TX process */
 141 #define TG3_TX_WAKEUP_THRESH(tnapi)             ((tnapi)->tx_pending / 4)
 142
 143 #define TG3_RAW_IP_ALIGN 2
 144
 145 /* number of ETHTOOL_GSTATS u64's */
 146 #define TG3_NUM_STATS           (sizeof(struct tg3_ethtool_stats)/sizeof(u64))
 147
 148 #define TG3_NUM_TEST            6
 149
 150 #define FIRMWARE_TG3            "tigon/tg3.bin"
 151 #define FIRMWARE_TG3TSO         "tigon/tg3_tso.bin"
 152 #define FIRMWARE_TG3TSO5        "tigon/tg3_tso5.bin"
 153
 154 static char version[] __devinitdata =
 155         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 156
 157 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 158 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 159 MODULE_LICENSE("GPL");
 160 MODULE_VERSION(DRV_MODULE_VERSION);
 161 MODULE_FIRMWARE(FIRMWARE_TG3);
 162 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
 163 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
 164
 165 #define TG3_RSS_MIN_NUM_MSIX_VECS       2
 166
 167 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 168 module_param(tg3_debug, int, 0);
 169 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 170
 171 static struct pci_device_id tg3_pci_tbl[] = {
 172         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
 173         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
 174         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
 175         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
 176         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
 177         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
 178         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
 179         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
 180         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
 181         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
 182         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
 183         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
 184         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
 185         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
 186         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
 187         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
 188         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
 189         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
 190         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
 191         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
 192         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
 193         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
 194         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
 195         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
 196         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
 197         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
 198         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
 199         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
 200         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
 201         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
 202         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
 203         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
 204         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
 205         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
 206         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
 207         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
 208         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
 209         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
 210         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
 211         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
 212         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
 213         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
 214         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
 215         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
 216         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
 217         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
 218         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
 219         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
 220         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
 221         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
 222         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
 223         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
 224         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
 225         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
 226         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
 227         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
 228         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
 229         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
 230         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
 231         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
 232         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_G)},
 233         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_F)},
 234         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
 235         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
 236         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790)},
 237         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)},
 238         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
 239         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
 240         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
 241         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
 242         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
 243         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
 244         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
 245         {}
 246 };
 247
 248 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 249
 250 static const struct {
 251         const char string[ETH_GSTRING_LEN];
 252 } ethtool_stats_keys[TG3_NUM_STATS] = {
 253         { "rx_octets" },
 254         { "rx_fragments" },
 255         { "rx_ucast_packets" },
 256         { "rx_mcast_packets" },
 257         { "rx_bcast_packets" },
 258         { "rx_fcs_errors" },
 259         { "rx_align_errors" },
 260         { "rx_xon_pause_rcvd" },
 261         { "rx_xoff_pause_rcvd" },
 262         { "rx_mac_ctrl_rcvd" },
 263         { "rx_xoff_entered" },
 264         { "rx_frame_too_long_errors" },
 265         { "rx_jabbers" },
 266         { "rx_undersize_packets" },
 267         { "rx_in_length_errors" },
 268         { "rx_out_length_errors" },
 269         { "rx_64_or_less_octet_packets" },
 270         { "rx_65_to_127_octet_packets" },
 271         { "rx_128_to_255_octet_packets" },
 272         { "rx_256_to_511_octet_packets" },
 273         { "rx_512_to_1023_octet_packets" },
 274         { "rx_1024_to_1522_octet_packets" },
 275         { "rx_1523_to_2047_octet_packets" },
 276         { "rx_2048_to_4095_octet_packets" },
 277         { "rx_4096_to_8191_octet_packets" },
 278         { "rx_8192_to_9022_octet_packets" },
 279
 280         { "tx_octets" },
 281         { "tx_collisions" },
 282
 283         { "tx_xon_sent" },
 284         { "tx_xoff_sent" },
 285         { "tx_flow_control" },
 286         { "tx_mac_errors" },
 287         { "tx_single_collisions" },
 288         { "tx_mult_collisions" },
 289         { "tx_deferred" },
 290         { "tx_excessive_collisions" },
 291         { "tx_late_collisions" },
 292         { "tx_collide_2times" },
 293         { "tx_collide_3times" },
 294         { "tx_collide_4times" },
 295         { "tx_collide_5times" },
 296         { "tx_collide_6times" },
 297         { "tx_collide_7times" },
 298         { "tx_collide_8times" },
 299         { "tx_collide_9times" },
 300         { "tx_collide_10times" },
 301         { "tx_collide_11times" },
 302         { "tx_collide_12times" },
 303         { "tx_collide_13times" },
 304         { "tx_collide_14times" },
 305         { "tx_collide_15times" },
 306         { "tx_ucast_packets" },
 307         { "tx_mcast_packets" },
 308         { "tx_bcast_packets" },
 309         { "tx_carrier_sense_errors" },
 310         { "tx_discards" },
 311         { "tx_errors" },
 312
 313         { "dma_writeq_full" },
 314         { "dma_write_prioq_full" },
 315         { "rxbds_empty" },
 316         { "rx_discards" },
 317         { "rx_errors" },
 318         { "rx_threshold_hit" },
 319
 320         { "dma_readq_full" },
 321         { "dma_read_prioq_full" },
 322         { "tx_comp_queue_full" },
 323
 324         { "ring_set_send_prod_index" },
 325         { "ring_status_update" },
 326         { "nic_irqs" },
 327         { "nic_avoided_irqs" },
 328         { "nic_tx_threshold_hit" }
 329 };
 330
 331 static const struct {
 332         const char string[ETH_GSTRING_LEN];
 333 } ethtool_test_keys[TG3_NUM_TEST] = {
 334         { "nvram test     (online) " },
 335         { "link test      (online) " },
 336         { "register test  (offline)" },
 337         { "memory test    (offline)" },
 338         { "loopback test  (offline)" },
 339         { "interrupt test (offline)" },
 340 };
 341
 342 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 343 {
 344         writel(val, tp->regs + off);
 345 }
 346
 347 static u32 tg3_read32(struct tg3 *tp, u32 off)
 348 {
 349         return (readl(tp->regs + off));
 350 }
 351
 352 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
 353 {
 354         writel(val, tp->aperegs + off);
 355 }
 356
 357 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
 358 {
 359         return (readl(tp->aperegs + off));
 360 }
 361
 362 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 363 {
 364         unsigned long flags;
 365
 366         spin_lock_irqsave(&tp->indirect_lock, flags);
 367         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 368         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 369         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 370 }
 371
 372 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 373 {
 374         writel(val, tp->regs + off);
 375         readl(tp->regs + off);
 376 }
 377
 378 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 379 {
 380         unsigned long flags;
 381         u32 val;
 382
 383         spin_lock_irqsave(&tp->indirect_lock, flags);
 384         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 385         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 386         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 387         return val;
 388 }
 389
 390 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 391 {
 392         unsigned long flags;
 393
 394         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 395                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 396                                        TG3_64BIT_REG_LOW, val);
 397                 return;
 398         }
 399         if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) {
 400                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 401                                        TG3_64BIT_REG_LOW, val);
 402                 return;
 403         }
 404
 405         spin_lock_irqsave(&tp->indirect_lock, flags);
 406         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 407         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 408         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 409
 410         /* In indirect mode when disabling interrupts, we also need
 411          * to clear the interrupt bit in the GRC local ctrl register.
 412          */
 413         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 414             (val == 0x1)) {
 415                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 416                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 417         }
 418 }
 419
 420 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 421 {
 422         unsigned long flags;
 423         u32 val;
 424
 425         spin_lock_irqsave(&tp->indirect_lock, flags);
 426         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 427         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 428         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 429         return val;
 430 }
 431
 432 /* usec_wait specifies the wait time in usec when writing to certain registers
 433  * where it is unsafe to read back the register without some delay.
 434  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 435  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 436  */
 437 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 438 {
 439         if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) ||
 440             (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 441                 /* Non-posted methods */
 442                 tp->write32(tp, off, val);
 443         else {
 444                 /* Posted method */
 445                 tg3_write32(tp, off, val);
 446                 if (usec_wait)
 447                         udelay(usec_wait);
 448                 tp->read32(tp, off);
 449         }
 450         /* Wait again after the read for the posted method to guarantee that
 451          * the wait time is met.
 452          */
 453         if (usec_wait)
 454                 udelay(usec_wait);
 455 }
 456
 457 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 458 {
 459         tp->write32_mbox(tp, off, val);
 460         if (!(tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) &&
 461             !(tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND))
 462                 tp->read32_mbox(tp, off);
 463 }
 464
 465 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 466 {
 467         void __iomem *mbox = tp->regs + off;
 468         writel(val, mbox);
 469         if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
 470                 writel(val, mbox);
 471         if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
 472                 readl(mbox);
 473 }
 474
 475 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
 476 {
 477         return (readl(tp->regs + off + GRCMBOX_BASE));
 478 }
 479
 480 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
 481 {
 482         writel(val, tp->regs + off + GRCMBOX_BASE);
 483 }
 484
 485 #define tw32_mailbox(reg, val)  tp->write32_mbox(tp, reg, val)
 486 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 487 #define tw32_rx_mbox(reg, val)  tp->write32_rx_mbox(tp, reg, val)
 488 #define tw32_tx_mbox(reg, val)  tp->write32_tx_mbox(tp, reg, val)
 489 #define tr32_mailbox(reg)       tp->read32_mbox(tp, reg)
 490
 491 #define tw32(reg,val)           tp->write32(tp, reg, val)
 492 #define tw32_f(reg,val)         _tw32_flush(tp,(reg),(val), 0)
 493 #define tw32_wait_f(reg,val,us) _tw32_flush(tp,(reg),(val), (us))
 494 #define tr32(reg)               tp->read32(tp, reg)
 495
 496 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 497 {
 498         unsigned long flags;
 499
 500         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 501             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
 502                 return;
 503
 504         spin_lock_irqsave(&tp->indirect_lock, flags);
 505         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 506                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 507                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 508
 509                 /* Always leave this as zero. */
 510                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 511         } else {
 512                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 513                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 514
 515                 /* Always leave this as zero. */
 516                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 517         }
 518         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 519 }
 520
 521 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 522 {
 523         unsigned long flags;
 524
 525         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 526             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
 527                 *val = 0;
 528                 return;
 529         }
 530
 531         spin_lock_irqsave(&tp->indirect_lock, flags);
 532         if (tp->tg3_flags & TG3_FLAG_SRAM_USE_CONFIG) {
 533                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 534                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 535
 536                 /* Always leave this as zero. */
 537                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 538         } else {
 539                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 540                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 541
 542                 /* Always leave this as zero. */
 543                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 544         }
 545         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 546 }
 547
 548 static void tg3_ape_lock_init(struct tg3 *tp)
 549 {
 550         int i;
 551
 552         /* Make sure the driver hasn't any stale locks. */
 553         for (i = 0; i < 8; i++)
 554                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + 4 * i,
 555                                 APE_LOCK_GRANT_DRIVER);
 556 }
 557
 558 static int tg3_ape_lock(struct tg3 *tp, int locknum)
 559 {
 560         int i, off;
 561         int ret = 0;
 562         u32 status;
 563
 564         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 565                 return 0;
 566
 567         switch (locknum) {
 568                 case TG3_APE_LOCK_GRC:
 569                 case TG3_APE_LOCK_MEM:
 570                         break;
 571                 default:
 572                         return -EINVAL;
 573         }
 574
 575         off = 4 * locknum;
 576
 577         tg3_ape_write32(tp, TG3_APE_LOCK_REQ + off, APE_LOCK_REQ_DRIVER);
 578
 579         /* Wait for up to 1 millisecond to acquire lock. */
 580         for (i = 0; i < 100; i++) {
 581                 status = tg3_ape_read32(tp, TG3_APE_LOCK_GRANT + off);
 582                 if (status == APE_LOCK_GRANT_DRIVER)
 583                         break;
 584                 udelay(10);
 585         }
 586
 587         if (status != APE_LOCK_GRANT_DRIVER) {
 588                 /* Revoke the lock request. */
 589                 tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off,
 590                                 APE_LOCK_GRANT_DRIVER);
 591
 592                 ret = -EBUSY;
 593         }
 594
 595         return ret;
 596 }
 597
 598 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
 599 {
 600         int off;
 601
 602         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
 603                 return;
 604
 605         switch (locknum) {
 606                 case TG3_APE_LOCK_GRC:
 607                 case TG3_APE_LOCK_MEM:
 608                         break;
 609                 default:
 610                         return;
 611         }
 612
 613         off = 4 * locknum;
 614         tg3_ape_write32(tp, TG3_APE_LOCK_GRANT + off, APE_LOCK_GRANT_DRIVER);
 615 }
 616
 617 static void tg3_disable_ints(struct tg3 *tp)
 618 {
 619         int i;
 620
 621         tw32(TG3PCI_MISC_HOST_CTRL,
 622              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 623         for (i = 0; i < tp->irq_max; i++)
 624                 tw32_mailbox_f(tp->napi[i].int_mbox, 0x00000001);
 625 }
 626
 627 static void tg3_enable_ints(struct tg3 *tp)
 628 {
 629         int i;
 630         u32 coal_now = 0;
 631
 632         tp->irq_sync = 0;
 633         wmb();
 634
 635         tw32(TG3PCI_MISC_HOST_CTRL,
 636              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 637
 638         for (i = 0; i < tp->irq_cnt; i++) {
 639                 struct tg3_napi *tnapi = &tp->napi[i];
 640                 tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
 641                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
 642                         tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
 643
 644                 coal_now |= tnapi->coal_now;
 645         }
 646
 647         /* Force an initial interrupt */
 648         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 649             (tp->napi[0].hw_status->status & SD_STATUS_UPDATED))
 650                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 651         else
 652                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 653                      HOSTCC_MODE_ENABLE | coal_now);
 654 }
 655
 656 static inline unsigned int tg3_has_work(struct tg3_napi *tnapi)
 657 {
 658         struct tg3 *tp = tnapi->tp;
 659         struct tg3_hw_status *sblk = tnapi->hw_status;
 660         unsigned int work_exists = 0;
 661
 662         /* check for phy events */
 663         if (!(tp->tg3_flags &
 664               (TG3_FLAG_USE_LINKCHG_REG |
 665                TG3_FLAG_POLL_SERDES))) {
 666                 if (sblk->status & SD_STATUS_LINK_CHG)
 667                         work_exists = 1;
 668         }
 669         /* check for RX/TX work to do */
 670         if (sblk->idx[0].tx_consumer != tnapi->tx_cons ||
 671             *(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
 672                 work_exists = 1;
 673
 674         return work_exists;
 675 }
 676
 677 /* tg3_int_reenable
 678  *  similar to tg3_enable_ints, but it accurately determines whether there
 679  *  is new work pending and can return without flushing the PIO write
 680  *  which reenables interrupts
 681  */
 682 static void tg3_int_reenable(struct tg3_napi *tnapi)
 683 {
 684         struct tg3 *tp = tnapi->tp;
 685
 686         tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
 687         mmiowb();
 688
 689         /* When doing tagged status, this work check is unnecessary.
 690          * The last_tag we write above tells the chip which piece of
 691          * work we've completed.
 692          */
 693         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) &&
 694             tg3_has_work(tnapi))
 695                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 696                      HOSTCC_MODE_ENABLE | tnapi->coal_now);
 697 }
 698
 699 static void tg3_napi_disable(struct tg3 *tp)
 700 {
 701         int i;
 702
 703         for (i = tp->irq_cnt - 1; i >= 0; i--)
 704                 napi_disable(&tp->napi[i].napi);
 705 }
 706
 707 static void tg3_napi_enable(struct tg3 *tp)
 708 {
 709         int i;
 710
 711         for (i = 0; i < tp->irq_cnt; i++)
 712                 napi_enable(&tp->napi[i].napi);
 713 }
 714
 715 static inline void tg3_netif_stop(struct tg3 *tp)
 716 {
 717         tp->dev->trans_start = jiffies; /* prevent tx timeout */
 718         tg3_napi_disable(tp);
 719         netif_tx_disable(tp->dev);
 720 }
 721
 722 static inline void tg3_netif_start(struct tg3 *tp)
 723 {
 724         /* NOTE: unconditional netif_tx_wake_all_queues is only
 725          * appropriate so long as all callers are assured to
 726          * have free tx slots (such as after tg3_init_hw)
 727          */
 728         netif_tx_wake_all_queues(tp->dev);
 729
 730         tg3_napi_enable(tp);
 731         tp->napi[0].hw_status->status |= SD_STATUS_UPDATED;
 732         tg3_enable_ints(tp);
 733 }
 734
 735 static void tg3_switch_clocks(struct tg3 *tp)
 736 {
 737         u32 clock_ctrl;
 738         u32 orig_clock_ctrl;
 739
 740         if ((tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
 741             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
 742                 return;
 743
 744         clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 745
 746         orig_clock_ctrl = clock_ctrl;
 747         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 748                        CLOCK_CTRL_CLKRUN_OENABLE |
 749                        0x1f);
 750         tp->pci_clock_ctrl = clock_ctrl;
 751
 752         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
 753                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 754                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 755                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 756                 }
 757         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 758                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 759                             clock_ctrl |
 760                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 761                             40);
 762                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 763                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 764                             40);
 765         }
 766         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 767 }
 768
 769 #define PHY_BUSY_LOOPS  5000
 770
 771 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 772 {
 773         u32 frame_val;
 774         unsigned int loops;
 775         int ret;
 776
 777         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 778                 tw32_f(MAC_MI_MODE,
 779                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 780                 udelay(80);
 781         }
 782
 783         *val = 0x0;
 784
 785         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
 786                       MI_COM_PHY_ADDR_MASK);
 787         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 788                       MI_COM_REG_ADDR_MASK);
 789         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 790
 791         tw32_f(MAC_MI_COM, frame_val);
 792
 793         loops = PHY_BUSY_LOOPS;
 794         while (loops != 0) {
 795                 udelay(10);
 796                 frame_val = tr32(MAC_MI_COM);
 797
 798                 if ((frame_val & MI_COM_BUSY) == 0) {
 799                         udelay(5);
 800                         frame_val = tr32(MAC_MI_COM);
 801                         break;
 802                 }
 803                 loops -= 1;
 804         }
 805
 806         ret = -EBUSY;
 807         if (loops != 0) {
 808                 *val = frame_val & MI_COM_DATA_MASK;
 809                 ret = 0;
 810         }
 811
 812         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 813                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 814                 udelay(80);
 815         }
 816
 817         return ret;
 818 }
 819
 820 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 821 {
 822         u32 frame_val;
 823         unsigned int loops;
 824         int ret;
 825
 826         if ((tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) &&
 827             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
 828                 return 0;
 829
 830         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 831                 tw32_f(MAC_MI_MODE,
 832                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 833                 udelay(80);
 834         }
 835
 836         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
 837                       MI_COM_PHY_ADDR_MASK);
 838         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 839                       MI_COM_REG_ADDR_MASK);
 840         frame_val |= (val & MI_COM_DATA_MASK);
 841         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 842
 843         tw32_f(MAC_MI_COM, frame_val);
 844
 845         loops = PHY_BUSY_LOOPS;
 846         while (loops != 0) {
 847                 udelay(10);
 848                 frame_val = tr32(MAC_MI_COM);
 849                 if ((frame_val & MI_COM_BUSY) == 0) {
 850                         udelay(5);
 851                         frame_val = tr32(MAC_MI_COM);
 852                         break;
 853                 }
 854                 loops -= 1;
 855         }
 856
 857         ret = -EBUSY;
 858         if (loops != 0)
 859                 ret = 0;
 860
 861         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 862                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 863                 udelay(80);
 864         }
 865
 866         return ret;
 867 }
 868
 869 static int tg3_bmcr_reset(struct tg3 *tp)
 870 {
 871         u32 phy_control;
 872         int limit, err;
 873
 874         /* OK, reset it, and poll the BMCR_RESET bit until it
 875          * clears or we time out.
 876          */
 877         phy_control = BMCR_RESET;
 878         err = tg3_writephy(tp, MII_BMCR, phy_control);
 879         if (err != 0)
 880                 return -EBUSY;
 881
 882         limit = 5000;
 883         while (limit--) {
 884                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
 885                 if (err != 0)
 886                         return -EBUSY;
 887
 888                 if ((phy_control & BMCR_RESET) == 0) {
 889                         udelay(40);
 890                         break;
 891                 }
 892                 udelay(10);
 893         }
 894         if (limit < 0)
 895                 return -EBUSY;
 896
 897         return 0;
 898 }
 899
 900 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
 901 {
 902         struct tg3 *tp = bp->priv;
 903         u32 val;
 904
 905         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_PAUSED)
 906                 return -EAGAIN;
 907
 908         if (tg3_readphy(tp, reg, &val))
 909                 return -EIO;
 910
 911         return val;
 912 }
 913
 914 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
 915 {
 916         struct tg3 *tp = bp->priv;
 917
 918         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_PAUSED)
 919                 return -EAGAIN;
 920
 921         if (tg3_writephy(tp, reg, val))
 922                 return -EIO;
 923
 924         return 0;
 925 }
 926
 927 static int tg3_mdio_reset(struct mii_bus *bp)
 928 {
 929         return 0;
 930 }
 931
 932 static void tg3_mdio_config_5785(struct tg3 *tp)
 933 {
 934         u32 val;
 935         struct phy_device *phydev;
 936
 937         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
 938         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
 939         case TG3_PHY_ID_BCM50610:
 940                 val = MAC_PHYCFG2_50610_LED_MODES;
 941                 break;
 942         case TG3_PHY_ID_BCMAC131:
 943                 val = MAC_PHYCFG2_AC131_LED_MODES;
 944                 break;
 945         case TG3_PHY_ID_RTL8211C:
 946                 val = MAC_PHYCFG2_RTL8211C_LED_MODES;
 947                 break;
 948         case TG3_PHY_ID_RTL8201E:
 949                 val = MAC_PHYCFG2_RTL8201E_LED_MODES;
 950                 break;
 951         default:
 952                 return;
 953         }
 954
 955         if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
 956                 tw32(MAC_PHYCFG2, val);
 957
 958                 val = tr32(MAC_PHYCFG1);
 959                 val &= ~(MAC_PHYCFG1_RGMII_INT |
 960                          MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK);
 961                 val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT;
 962                 tw32(MAC_PHYCFG1, val);
 963
 964                 return;
 965         }
 966
 967         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE))
 968                 val |= MAC_PHYCFG2_EMODE_MASK_MASK |
 969                        MAC_PHYCFG2_FMODE_MASK_MASK |
 970                        MAC_PHYCFG2_GMODE_MASK_MASK |
 971                        MAC_PHYCFG2_ACT_MASK_MASK   |
 972                        MAC_PHYCFG2_QUAL_MASK_MASK |
 973                        MAC_PHYCFG2_INBAND_ENABLE;
 974
 975         tw32(MAC_PHYCFG2, val);
 976
 977         val = tr32(MAC_PHYCFG1);
 978         val &= ~(MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK |
 979                  MAC_PHYCFG1_RGMII_EXT_RX_DEC | MAC_PHYCFG1_RGMII_SND_STAT_EN);
 980         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)) {
 981                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
 982                         val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
 983                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
 984                         val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
 985         }
 986         val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT |
 987                MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV;
 988         tw32(MAC_PHYCFG1, val);
 989
 990         val = tr32(MAC_EXT_RGMII_MODE);
 991         val &= ~(MAC_RGMII_MODE_RX_INT_B |
 992                  MAC_RGMII_MODE_RX_QUALITY |
 993                  MAC_RGMII_MODE_RX_ACTIVITY |
 994                  MAC_RGMII_MODE_RX_ENG_DET |
 995                  MAC_RGMII_MODE_TX_ENABLE |
 996                  MAC_RGMII_MODE_TX_LOWPWR |
 997                  MAC_RGMII_MODE_TX_RESET);
 998         if (!(tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)) {
 999                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
1000                         val |= MAC_RGMII_MODE_RX_INT_B |
1001                                MAC_RGMII_MODE_RX_QUALITY |
1002                                MAC_RGMII_MODE_RX_ACTIVITY |
1003                                MAC_RGMII_MODE_RX_ENG_DET;
1004                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
1005                         val |= MAC_RGMII_MODE_TX_ENABLE |
1006                                MAC_RGMII_MODE_TX_LOWPWR |
1007                                MAC_RGMII_MODE_TX_RESET;
1008         }
1009         tw32(MAC_EXT_RGMII_MODE, val);
1010 }
1011
1012 static void tg3_mdio_start(struct tg3 *tp)
1013 {
1014         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) {
1015                 mutex_lock(&tp->mdio_bus->mdio_lock);
1016                 tp->tg3_flags3 &= ~TG3_FLG3_MDIOBUS_PAUSED;
1017                 mutex_unlock(&tp->mdio_bus->mdio_lock);
1018         }
1019
1020         tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
1021         tw32_f(MAC_MI_MODE, tp->mi_mode);
1022         udelay(80);
1023
1024         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
1025                 u32 funcnum, is_serdes;
1026
1027                 funcnum = tr32(TG3_CPMU_STATUS) & TG3_CPMU_STATUS_PCIE_FUNC;
1028                 if (funcnum)
1029                         tp->phy_addr = 2;
1030                 else
1031                         tp->phy_addr = 1;
1032
1033                 is_serdes = tr32(SG_DIG_STATUS) & SG_DIG_IS_SERDES;
1034                 if (is_serdes)
1035                         tp->phy_addr += 7;
1036         } else
1037                 tp->phy_addr = PHY_ADDR;
1038
1039         if ((tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) &&
1040             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1041                 tg3_mdio_config_5785(tp);
1042 }
1043
1044 static void tg3_mdio_stop(struct tg3 *tp)
1045 {
1046         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) {
1047                 mutex_lock(&tp->mdio_bus->mdio_lock);
1048                 tp->tg3_flags3 |= TG3_FLG3_MDIOBUS_PAUSED;
1049                 mutex_unlock(&tp->mdio_bus->mdio_lock);
1050         }
1051 }
1052
1053 static int tg3_mdio_init(struct tg3 *tp)
1054 {
1055         int i;
1056         u32 reg;
1057         struct phy_device *phydev;
1058
1059         tg3_mdio_start(tp);
1060
1061         if (!(tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) ||
1062             (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED))
1063                 return 0;
1064
1065         tp->mdio_bus = mdiobus_alloc();
1066         if (tp->mdio_bus == NULL)
1067                 return -ENOMEM;
1068
1069         tp->mdio_bus->name     = "tg3 mdio bus";
1070         snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1071                  (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1072         tp->mdio_bus->priv     = tp;
1073         tp->mdio_bus->parent   = &tp->pdev->dev;
1074         tp->mdio_bus->read     = &tg3_mdio_read;
1075         tp->mdio_bus->write    = &tg3_mdio_write;
1076         tp->mdio_bus->reset    = &tg3_mdio_reset;
1077         tp->mdio_bus->phy_mask = ~(1 << PHY_ADDR);
1078         tp->mdio_bus->irq      = &tp->mdio_irq[0];
1079
1080         for (i = 0; i < PHY_MAX_ADDR; i++)
1081                 tp->mdio_bus->irq[i] = PHY_POLL;
1082
1083         /* The bus registration will look for all the PHYs on the mdio bus.
1084          * Unfortunately, it does not ensure the PHY is powered up before
1085          * accessing the PHY ID registers.  A chip reset is the
1086          * quickest way to bring the device back to an operational state..
1087          */
1088         if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1089                 tg3_bmcr_reset(tp);
1090
1091         i = mdiobus_register(tp->mdio_bus);
1092         if (i) {
1093                 printk(KERN_WARNING "%s: mdiobus_reg failed (0x%x)\n",
1094                         tp->dev->name, i);
1095                 mdiobus_free(tp->mdio_bus);
1096                 return i;
1097         }
1098
1099         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1100
1101         if (!phydev || !phydev->drv) {
1102                 printk(KERN_WARNING "%s: No PHY devices\n", tp->dev->name);
1103                 mdiobus_unregister(tp->mdio_bus);
1104                 mdiobus_free(tp->mdio_bus);
1105                 return -ENODEV;
1106         }
1107
1108         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1109         case TG3_PHY_ID_BCM57780:
1110                 phydev->interface = PHY_INTERFACE_MODE_GMII;
1111                 break;
1112         case TG3_PHY_ID_BCM50610:
1113                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE)
1114                         phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE;
1115                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN)
1116                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE;
1117                 if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_TX_EN)
1118                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE;
1119                 /* fallthru */
1120         case TG3_PHY_ID_RTL8211C:
1121                 phydev->interface = PHY_INTERFACE_MODE_RGMII;
1122                 break;
1123         case TG3_PHY_ID_RTL8201E:
1124         case TG3_PHY_ID_BCMAC131:
1125                 phydev->interface = PHY_INTERFACE_MODE_MII;
1126                 tp->tg3_flags3 |= TG3_FLG3_PHY_IS_FET;
1127                 break;
1128         }
1129
1130         tp->tg3_flags3 |= TG3_FLG3_MDIOBUS_INITED;
1131
1132         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1133                 tg3_mdio_config_5785(tp);
1134
1135         return 0;
1136 }
1137
1138 static void tg3_mdio_fini(struct tg3 *tp)
1139 {
1140         if (tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) {
1141                 tp->tg3_flags3 &= ~TG3_FLG3_MDIOBUS_INITED;
1142                 mdiobus_unregister(tp->mdio_bus);
1143                 mdiobus_free(tp->mdio_bus);
1144                 tp->tg3_flags3 &= ~TG3_FLG3_MDIOBUS_PAUSED;
1145         }
1146 }
1147
1148 /* tp->lock is held. */
1149 static inline void tg3_generate_fw_event(struct tg3 *tp)
1150 {
1151         u32 val;
1152
1153         val = tr32(GRC_RX_CPU_EVENT);
1154         val |= GRC_RX_CPU_DRIVER_EVENT;
1155         tw32_f(GRC_RX_CPU_EVENT, val);
1156
1157         tp->last_event_jiffies = jiffies;
1158 }
1159
1160 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1161
1162 /* tp->lock is held. */
1163 static void tg3_wait_for_event_ack(struct tg3 *tp)
1164 {
1165         int i;
1166         unsigned int delay_cnt;
1167         long time_remain;
1168
1169         /* If enough time has passed, no wait is necessary. */
1170         time_remain = (long)(tp->last_event_jiffies + 1 +
1171                       usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1172                       (long)jiffies;
1173         if (time_remain < 0)
1174                 return;
1175
1176         /* Check if we can shorten the wait time. */
1177         delay_cnt = jiffies_to_usecs(time_remain);
1178         if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1179                 delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1180         delay_cnt = (delay_cnt >> 3) + 1;
1181
1182         for (i = 0; i < delay_cnt; i++) {
1183                 if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1184                         break;
1185                 udelay(8);
1186         }
1187 }
1188
1189 /* tp->lock is held. */
1190 static void tg3_ump_link_report(struct tg3 *tp)
1191 {
1192         u32 reg;
1193         u32 val;
1194
1195         if (!(tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
1196             !(tp->tg3_flags  & TG3_FLAG_ENABLE_ASF))
1197                 return;
1198
1199         tg3_wait_for_event_ack(tp);
1200
1201         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1202
1203         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1204
1205         val = 0;
1206         if (!tg3_readphy(tp, MII_BMCR, &reg))
1207                 val = reg << 16;
1208         if (!tg3_readphy(tp, MII_BMSR, &reg))
1209                 val |= (reg & 0xffff);
1210         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, val);
1211
1212         val = 0;
1213         if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1214                 val = reg << 16;
1215         if (!tg3_readphy(tp, MII_LPA, &reg))
1216                 val |= (reg & 0xffff);
1217         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 4, val);
1218
1219         val = 0;
1220         if (!(tp->tg3_flags2 & TG3_FLG2_MII_SERDES)) {
1221                 if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1222                         val = reg << 16;
1223                 if (!tg3_readphy(tp, MII_STAT1000, &reg))
1224                         val |= (reg & 0xffff);
1225         }
1226         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 8, val);
1227
1228         if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1229                 val = reg << 16;
1230         else
1231                 val = 0;
1232         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 12, val);
1233
1234         tg3_generate_fw_event(tp);
1235 }
1236
1237 static void tg3_link_report(struct tg3 *tp)
1238 {
1239         if (!netif_carrier_ok(tp->dev)) {
1240                 if (netif_msg_link(tp))
1241                         printk(KERN_INFO PFX "%s: Link is down.\n",
1242                                tp->dev->name);
1243                 tg3_ump_link_report(tp);
1244         } else if (netif_msg_link(tp)) {
1245                 printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
1246                        tp->dev->name,
1247                        (tp->link_config.active_speed == SPEED_1000 ?
1248                         1000 :
1249                         (tp->link_config.active_speed == SPEED_100 ?
1250                          100 : 10)),
1251                        (tp->link_config.active_duplex == DUPLEX_FULL ?
1252                         "full" : "half"));
1253
1254                 printk(KERN_INFO PFX
1255                        "%s: Flow control is %s for TX and %s for RX.\n",
1256                        tp->dev->name,
1257                        (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1258                        "on" : "off",
1259                        (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1260                        "on" : "off");
1261                 tg3_ump_link_report(tp);
1262         }
1263 }
1264
1265 static u16 tg3_advert_flowctrl_1000T(u8 flow_ctrl)
1266 {
1267         u16 miireg;
1268
1269         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1270                 miireg = ADVERTISE_PAUSE_CAP;
1271         else if (flow_ctrl & FLOW_CTRL_TX)
1272                 miireg = ADVERTISE_PAUSE_ASYM;
1273         else if (flow_ctrl & FLOW_CTRL_RX)
1274                 miireg = ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
1275         else
1276                 miireg = 0;
1277
1278         return miireg;
1279 }
1280
1281 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1282 {
1283         u16 miireg;
1284
1285         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1286                 miireg = ADVERTISE_1000XPAUSE;
1287         else if (flow_ctrl & FLOW_CTRL_TX)
1288                 miireg = ADVERTISE_1000XPSE_ASYM;
1289         else if (flow_ctrl & FLOW_CTRL_RX)
1290                 miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1291         else
1292                 miireg = 0;
1293
1294         return miireg;
1295 }
1296
1297 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1298 {
1299         u8 cap = 0;
1300
1301         if (lcladv & ADVERTISE_1000XPAUSE) {
1302                 if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1303                         if (rmtadv & LPA_1000XPAUSE)
1304                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1305                         else if (rmtadv & LPA_1000XPAUSE_ASYM)
1306                                 cap = FLOW_CTRL_RX;
1307                 } else {
1308                         if (rmtadv & LPA_1000XPAUSE)
1309                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1310                 }
1311         } else if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1312                 if ((rmtadv & LPA_1000XPAUSE) && (rmtadv & LPA_1000XPAUSE_ASYM))
1313                         cap = FLOW_CTRL_TX;
1314         }
1315
1316         return cap;
1317 }
1318
1319 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1320 {
1321         u8 autoneg;
1322         u8 flowctrl = 0;
1323         u32 old_rx_mode = tp->rx_mode;
1324         u32 old_tx_mode = tp->tx_mode;
1325
1326         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB)
1327                 autoneg = tp->mdio_bus->phy_map[PHY_ADDR]->autoneg;
1328         else
1329                 autoneg = tp->link_config.autoneg;
1330
1331         if (autoneg == AUTONEG_ENABLE &&
1332             (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)) {
1333                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
1334                         flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1335                 else
1336                         flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1337         } else
1338                 flowctrl = tp->link_config.flowctrl;
1339
1340         tp->link_config.active_flowctrl = flowctrl;
1341
1342         if (flowctrl & FLOW_CTRL_RX)
1343                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1344         else
1345                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1346
1347         if (old_rx_mode != tp->rx_mode)
1348                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1349
1350         if (flowctrl & FLOW_CTRL_TX)
1351                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1352         else
1353                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1354
1355         if (old_tx_mode != tp->tx_mode)
1356                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1357 }
1358
1359 static void tg3_adjust_link(struct net_device *dev)
1360 {
1361         u8 oldflowctrl, linkmesg = 0;
1362         u32 mac_mode, lcl_adv, rmt_adv;
1363         struct tg3 *tp = netdev_priv(dev);
1364         struct phy_device *phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1365
1366         spin_lock(&tp->lock);
1367
1368         mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
1369                                     MAC_MODE_HALF_DUPLEX);
1370
1371         oldflowctrl = tp->link_config.active_flowctrl;
1372
1373         if (phydev->link) {
1374                 lcl_adv = 0;
1375                 rmt_adv = 0;
1376
1377                 if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
1378                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1379                 else
1380                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
1381
1382                 if (phydev->duplex == DUPLEX_HALF)
1383                         mac_mode |= MAC_MODE_HALF_DUPLEX;
1384                 else {
1385                         lcl_adv = tg3_advert_flowctrl_1000T(
1386                                   tp->link_config.flowctrl);
1387
1388                         if (phydev->pause)
1389                                 rmt_adv = LPA_PAUSE_CAP;
1390                         if (phydev->asym_pause)
1391                                 rmt_adv |= LPA_PAUSE_ASYM;
1392                 }
1393
1394                 tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
1395         } else
1396                 mac_mode |= MAC_MODE_PORT_MODE_GMII;
1397
1398         if (mac_mode != tp->mac_mode) {
1399                 tp->mac_mode = mac_mode;
1400                 tw32_f(MAC_MODE, tp->mac_mode);
1401                 udelay(40);
1402         }
1403
1404         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) {
1405                 if (phydev->speed == SPEED_10)
1406                         tw32(MAC_MI_STAT,
1407                              MAC_MI_STAT_10MBPS_MODE |
1408                              MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1409                 else
1410                         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1411         }
1412
1413         if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
1414                 tw32(MAC_TX_LENGTHS,
1415                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1416                       (6 << TX_LENGTHS_IPG_SHIFT) |
1417                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
1418         else
1419                 tw32(MAC_TX_LENGTHS,
1420                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1421                       (6 << TX_LENGTHS_IPG_SHIFT) |
1422                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
1423
1424         if ((phydev->link && tp->link_config.active_speed == SPEED_INVALID) ||
1425             (!phydev->link && tp->link_config.active_speed != SPEED_INVALID) ||
1426             phydev->speed != tp->link_config.active_speed ||
1427             phydev->duplex != tp->link_config.active_duplex ||
1428             oldflowctrl != tp->link_config.active_flowctrl)
1429             linkmesg = 1;
1430
1431         tp->link_config.active_speed = phydev->speed;
1432         tp->link_config.active_duplex = phydev->duplex;
1433
1434         spin_unlock(&tp->lock);
1435
1436         if (linkmesg)
1437                 tg3_link_report(tp);
1438 }
1439
1440 static int tg3_phy_init(struct tg3 *tp)
1441 {
1442         struct phy_device *phydev;
1443
1444         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED)
1445                 return 0;
1446
1447         /* Bring the PHY back to a known state. */
1448         tg3_bmcr_reset(tp);
1449
1450         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1451
1452         /* Attach the MAC to the PHY. */
1453         phydev = phy_connect(tp->dev, dev_name(&phydev->dev), tg3_adjust_link,
1454                              phydev->dev_flags, phydev->interface);
1455         if (IS_ERR(phydev)) {
1456                 printk(KERN_ERR "%s: Could not attach to PHY\n", tp->dev->name);
1457                 return PTR_ERR(phydev);
1458         }
1459
1460         /* Mask with MAC supported features. */
1461         switch (phydev->interface) {
1462         case PHY_INTERFACE_MODE_GMII:
1463         case PHY_INTERFACE_MODE_RGMII:
1464                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
1465                         phydev->supported &= (PHY_GBIT_FEATURES |
1466                                               SUPPORTED_Pause |
1467                                               SUPPORTED_Asym_Pause);
1468                         break;
1469                 }
1470                 /* fallthru */
1471         case PHY_INTERFACE_MODE_MII:
1472                 phydev->supported &= (PHY_BASIC_FEATURES |
1473                                       SUPPORTED_Pause |
1474                                       SUPPORTED_Asym_Pause);
1475                 break;
1476         default:
1477                 phy_disconnect(tp->mdio_bus->phy_map[PHY_ADDR]);
1478                 return -EINVAL;
1479         }
1480
1481         tp->tg3_flags3 |= TG3_FLG3_PHY_CONNECTED;
1482
1483         phydev->advertising = phydev->supported;
1484
1485         return 0;
1486 }
1487
1488 static void tg3_phy_start(struct tg3 *tp)
1489 {
1490         struct phy_device *phydev;
1491
1492         if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
1493                 return;
1494
1495         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
1496
1497         if (tp->link_config.phy_is_low_power) {
1498                 tp->link_config.phy_is_low_power = 0;
1499                 phydev->speed = tp->link_config.orig_speed;
1500                 phydev->duplex = tp->link_config.orig_duplex;
1501                 phydev->autoneg = tp->link_config.orig_autoneg;
1502                 phydev->advertising = tp->link_config.orig_advertising;
1503         }
1504
1505         phy_start(phydev);
1506
1507         phy_start_aneg(phydev);
1508 }
1509
1510 static void tg3_phy_stop(struct tg3 *tp)
1511 {
1512         if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
1513                 return;
1514
1515         phy_stop(tp->mdio_bus->phy_map[PHY_ADDR]);
1516 }
1517
1518 static void tg3_phy_fini(struct tg3 *tp)
1519 {
1520         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) {
1521                 phy_disconnect(tp->mdio_bus->phy_map[PHY_ADDR]);
1522                 tp->tg3_flags3 &= ~TG3_FLG3_PHY_CONNECTED;
1523         }
1524 }
1525
1526 static void tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
1527 {
1528         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
1529         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
1530 }
1531
1532 static void tg3_phy_fet_toggle_apd(struct tg3 *tp, bool enable)
1533 {
1534         u32 phytest;
1535
1536         if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
1537                 u32 phy;
1538
1539                 tg3_writephy(tp, MII_TG3_FET_TEST,
1540                              phytest | MII_TG3_FET_SHADOW_EN);
1541                 if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXSTAT2, &phy)) {
1542                         if (enable)
1543                                 phy |= MII_TG3_FET_SHDW_AUXSTAT2_APD;
1544                         else
1545                                 phy &= ~MII_TG3_FET_SHDW_AUXSTAT2_APD;
1546                         tg3_writephy(tp, MII_TG3_FET_SHDW_AUXSTAT2, phy);
1547                 }
1548                 tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
1549         }
1550 }
1551
1552 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
1553 {
1554         u32 reg;
1555
1556         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
1557                 return;
1558
1559         if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
1560                 tg3_phy_fet_toggle_apd(tp, enable);
1561                 return;
1562         }
1563
1564         reg = MII_TG3_MISC_SHDW_WREN |
1565               MII_TG3_MISC_SHDW_SCR5_SEL |
1566               MII_TG3_MISC_SHDW_SCR5_LPED |
1567               MII_TG3_MISC_SHDW_SCR5_DLPTLM |
1568               MII_TG3_MISC_SHDW_SCR5_SDTL |
1569               MII_TG3_MISC_SHDW_SCR5_C125OE;
1570         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 || !enable)
1571                 reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
1572
1573         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1574
1575
1576         reg = MII_TG3_MISC_SHDW_WREN |
1577               MII_TG3_MISC_SHDW_APD_SEL |
1578               MII_TG3_MISC_SHDW_APD_WKTM_84MS;
1579         if (enable)
1580                 reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
1581
1582         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1583 }
1584
1585 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
1586 {
1587         u32 phy;
1588
1589         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
1590             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
1591                 return;
1592
1593         if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
1594                 u32 ephy;
1595
1596                 if (!tg3_readphy(tp, MII_TG3_FET_TEST, &ephy)) {
1597                         u32 reg = MII_TG3_FET_SHDW_MISCCTRL;
1598
1599                         tg3_writephy(tp, MII_TG3_FET_TEST,
1600                                      ephy | MII_TG3_FET_SHADOW_EN);
1601                         if (!tg3_readphy(tp, reg, &phy)) {
1602                                 if (enable)
1603                                         phy |= MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1604                                 else
1605                                         phy &= ~MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1606                                 tg3_writephy(tp, reg, phy);
1607                         }
1608                         tg3_writephy(tp, MII_TG3_FET_TEST, ephy);
1609                 }
1610         } else {
1611                 phy = MII_TG3_AUXCTL_MISC_RDSEL_MISC |
1612                       MII_TG3_AUXCTL_SHDWSEL_MISC;
1613                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, phy) &&
1614                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy)) {
1615                         if (enable)
1616                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1617                         else
1618                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1619                         phy |= MII_TG3_AUXCTL_MISC_WREN;
1620                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1621                 }
1622         }
1623 }
1624
1625 static void tg3_phy_set_wirespeed(struct tg3 *tp)
1626 {
1627         u32 val;
1628
1629         if (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED)
1630                 return;
1631
1632         if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x7007) &&
1633             !tg3_readphy(tp, MII_TG3_AUX_CTRL, &val))
1634                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
1635                              (val | (1 << 15) | (1 << 4)));
1636 }
1637
1638 static void tg3_phy_apply_otp(struct tg3 *tp)
1639 {
1640         u32 otp, phy;
1641
1642         if (!tp->phy_otp)
1643                 return;
1644
1645         otp = tp->phy_otp;
1646
1647         /* Enable SM_DSP clock and tx 6dB coding. */
1648         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
1649               MII_TG3_AUXCTL_ACTL_SMDSP_ENA |
1650               MII_TG3_AUXCTL_ACTL_TX_6DB;
1651         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1652
1653         phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
1654         phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
1655         tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
1656
1657         phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
1658               ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
1659         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
1660
1661         phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
1662         phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
1663         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
1664
1665         phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
1666         tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
1667
1668         phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
1669         tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
1670
1671         phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
1672               ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
1673         tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
1674
1675         /* Turn off SM_DSP clock. */
1676         phy = MII_TG3_AUXCTL_SHDWSEL_AUXCTL |
1677               MII_TG3_AUXCTL_ACTL_TX_6DB;
1678         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy);
1679 }
1680
1681 static int tg3_wait_macro_done(struct tg3 *tp)
1682 {
1683         int limit = 100;
1684
1685         while (limit--) {
1686                 u32 tmp32;
1687
1688                 if (!tg3_readphy(tp, 0x16, &tmp32)) {
1689                         if ((tmp32 & 0x1000) == 0)
1690                                 break;
1691                 }
1692         }
1693         if (limit < 0)
1694                 return -EBUSY;
1695
1696         return 0;
1697 }
1698
1699 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
1700 {
1701         static const u32 test_pat[4][6] = {
1702         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
1703         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
1704         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
1705         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
1706         };
1707         int chan;
1708
1709         for (chan = 0; chan < 4; chan++) {
1710                 int i;
1711
1712                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1713                              (chan * 0x2000) | 0x0200);
1714                 tg3_writephy(tp, 0x16, 0x0002);
1715
1716                 for (i = 0; i < 6; i++)
1717                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
1718                                      test_pat[chan][i]);
1719
1720                 tg3_writephy(tp, 0x16, 0x0202);
1721                 if (tg3_wait_macro_done(tp)) {
1722                         *resetp = 1;
1723                         return -EBUSY;
1724                 }
1725
1726                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1727                              (chan * 0x2000) | 0x0200);
1728                 tg3_writephy(tp, 0x16, 0x0082);
1729                 if (tg3_wait_macro_done(tp)) {
1730                         *resetp = 1;
1731                         return -EBUSY;
1732                 }
1733
1734                 tg3_writephy(tp, 0x16, 0x0802);
1735                 if (tg3_wait_macro_done(tp)) {
1736                         *resetp = 1;
1737                         return -EBUSY;
1738                 }
1739
1740                 for (i = 0; i < 6; i += 2) {
1741                         u32 low, high;
1742
1743                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
1744                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
1745                             tg3_wait_macro_done(tp)) {
1746                                 *resetp = 1;
1747                                 return -EBUSY;
1748                         }
1749                         low &= 0x7fff;
1750                         high &= 0x000f;
1751                         if (low != test_pat[chan][i] ||
1752                             high != test_pat[chan][i+1]) {
1753                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
1754                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
1755                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
1756
1757                                 return -EBUSY;
1758                         }
1759                 }
1760         }
1761
1762         return 0;
1763 }
1764
1765 static int tg3_phy_reset_chanpat(struct tg3 *tp)
1766 {
1767         int chan;
1768
1769         for (chan = 0; chan < 4; chan++) {
1770                 int i;
1771
1772                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1773                              (chan * 0x2000) | 0x0200);
1774                 tg3_writephy(tp, 0x16, 0x0002);
1775                 for (i = 0; i < 6; i++)
1776                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
1777                 tg3_writephy(tp, 0x16, 0x0202);
1778                 if (tg3_wait_macro_done(tp))
1779                         return -EBUSY;
1780         }
1781
1782         return 0;
1783 }
1784
1785 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
1786 {
1787         u32 reg32, phy9_orig;
1788         int retries, do_phy_reset, err;
1789
1790         retries = 10;
1791         do_phy_reset = 1;
1792         do {
1793                 if (do_phy_reset) {
1794                         err = tg3_bmcr_reset(tp);
1795                         if (err)
1796                                 return err;
1797                         do_phy_reset = 0;
1798                 }
1799
1800                 /* Disable transmitter and interrupt.  */
1801                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
1802                         continue;
1803
1804                 reg32 |= 0x3000;
1805                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1806
1807                 /* Set full-duplex, 1000 mbps.  */
1808                 tg3_writephy(tp, MII_BMCR,
1809                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
1810
1811                 /* Set to master mode.  */
1812                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
1813                         continue;
1814
1815                 tg3_writephy(tp, MII_TG3_CTRL,
1816                              (MII_TG3_CTRL_AS_MASTER |
1817                               MII_TG3_CTRL_ENABLE_AS_MASTER));
1818
1819                 /* Enable SM_DSP_CLOCK and 6dB.  */
1820                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1821
1822                 /* Block the PHY control access.  */
1823                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1824                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0800);
1825
1826                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
1827                 if (!err)
1828                         break;
1829         } while (--retries);
1830
1831         err = tg3_phy_reset_chanpat(tp);
1832         if (err)
1833                 return err;
1834
1835         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8005);
1836         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0000);
1837
1838         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
1839         tg3_writephy(tp, 0x16, 0x0000);
1840
1841         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1842             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
1843                 /* Set Extended packet length bit for jumbo frames */
1844                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4400);
1845         }
1846         else {
1847                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1848         }
1849
1850         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
1851
1852         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
1853                 reg32 &= ~0x3000;
1854                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1855         } else if (!err)
1856                 err = -EBUSY;
1857
1858         return err;
1859 }
1860
1861 /* This will reset the tigon3 PHY if there is no valid
1862  * link unless the FORCE argument is non-zero.
1863  */
1864 static int tg3_phy_reset(struct tg3 *tp)
1865 {
1866         u32 cpmuctrl;
1867         u32 phy_status;
1868         int err;
1869
1870         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1871                 u32 val;
1872
1873                 val = tr32(GRC_MISC_CFG);
1874                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
1875                 udelay(40);
1876         }
1877         err  = tg3_readphy(tp, MII_BMSR, &phy_status);
1878         err |= tg3_readphy(tp, MII_BMSR, &phy_status);
1879         if (err != 0)
1880                 return -EBUSY;
1881
1882         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
1883                 netif_carrier_off(tp->dev);
1884                 tg3_link_report(tp);
1885         }
1886
1887         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
1888             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
1889             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
1890                 err = tg3_phy_reset_5703_4_5(tp);
1891                 if (err)
1892                         return err;
1893                 goto out;
1894         }
1895
1896         cpmuctrl = 0;
1897         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
1898             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
1899                 cpmuctrl = tr32(TG3_CPMU_CTRL);
1900                 if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
1901                         tw32(TG3_CPMU_CTRL,
1902                              cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
1903         }
1904
1905         err = tg3_bmcr_reset(tp);
1906         if (err)
1907                 return err;
1908
1909         if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
1910                 u32 phy;
1911
1912                 phy = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
1913                 tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, phy);
1914
1915                 tw32(TG3_CPMU_CTRL, cpmuctrl);
1916         }
1917
1918         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
1919             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
1920                 u32 val;
1921
1922                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
1923                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
1924                     CPMU_LSPD_1000MB_MACCLK_12_5) {
1925                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
1926                         udelay(40);
1927                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
1928                 }
1929         }
1930
1931         tg3_phy_apply_otp(tp);
1932
1933         if (tp->tg3_flags3 & TG3_FLG3_PHY_ENABLE_APD)
1934                 tg3_phy_toggle_apd(tp, true);
1935         else
1936                 tg3_phy_toggle_apd(tp, false);
1937
1938 out:
1939         if (tp->tg3_flags2 & TG3_FLG2_PHY_ADC_BUG) {
1940                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1941                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1942                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x2aaa);
1943                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1944                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0323);
1945                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1946         }
1947         if (tp->tg3_flags2 & TG3_FLG2_PHY_5704_A0_BUG) {
1948                 tg3_writephy(tp, 0x1c, 0x8d68);
1949                 tg3_writephy(tp, 0x1c, 0x8d68);
1950         }
1951         if (tp->tg3_flags2 & TG3_FLG2_PHY_BER_BUG) {
1952                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1953                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1954                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x310b);
1955                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
1956                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x9506);
1957                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x401f);
1958                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x14e2);
1959                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1960         }
1961         else if (tp->tg3_flags2 & TG3_FLG2_PHY_JITTER_BUG) {
1962                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c00);
1963                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
1964                 if (tp->tg3_flags2 & TG3_FLG2_PHY_ADJUST_TRIM) {
1965                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
1966                         tg3_writephy(tp, MII_TG3_TEST1,
1967                                      MII_TG3_TEST1_TRIM_EN | 0x4);
1968                 } else
1969                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
1970                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0400);
1971         }
1972         /* Set Extended packet length bit (bit 14) on all chips that */
1973         /* support jumbo frames */
1974         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
1975                 /* Cannot do read-modify-write on 5401 */
1976                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
1977         } else if (tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) {
1978                 u32 phy_reg;
1979
1980                 /* Set bit 14 with read-modify-write to preserve other bits */
1981                 if (!tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0007) &&
1982                     !tg3_readphy(tp, MII_TG3_AUX_CTRL, &phy_reg))
1983                         tg3_writephy(tp, MII_TG3_AUX_CTRL, phy_reg | 0x4000);
1984         }
1985
1986         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
1987          * jumbo frames transmission.
1988          */
1989         if (tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) {
1990                 u32 phy_reg;
1991
1992                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &phy_reg))
1993                     tg3_writephy(tp, MII_TG3_EXT_CTRL,
1994                                  phy_reg | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
1995         }
1996
1997         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
1998                 /* adjust output voltage */
1999                 tg3_writephy(tp, MII_TG3_FET_PTEST, 0x12);
2000         }
2001
2002         tg3_phy_toggle_automdix(tp, 1);
2003         tg3_phy_set_wirespeed(tp);
2004         return 0;
2005 }
2006
2007 static void tg3_frob_aux_power(struct tg3 *tp)
2008 {
2009         struct tg3 *tp_peer = tp;
2010
2011         if ((tp->tg3_flags2 & TG3_FLG2_IS_NIC) == 0)
2012                 return;
2013
2014         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2015             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 ||
2016             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
2017                 struct net_device *dev_peer;
2018
2019                 dev_peer = pci_get_drvdata(tp->pdev_peer);
2020                 /* remove_one() may have been run on the peer. */
2021                 if (!dev_peer)
2022                         tp_peer = tp;
2023                 else
2024                         tp_peer = netdev_priv(dev_peer);
2025         }
2026
2027         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
2028             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0 ||
2029             (tp_peer->tg3_flags & TG3_FLAG_WOL_ENABLE) != 0 ||
2030             (tp_peer->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
2031                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2032                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2033                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2034                                     (GRC_LCLCTRL_GPIO_OE0 |
2035                                      GRC_LCLCTRL_GPIO_OE1 |
2036                                      GRC_LCLCTRL_GPIO_OE2 |
2037                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
2038                                      GRC_LCLCTRL_GPIO_OUTPUT1),
2039                                     100);
2040                 } else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
2041                            tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
2042                         /* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
2043                         u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
2044                                              GRC_LCLCTRL_GPIO_OE1 |
2045                                              GRC_LCLCTRL_GPIO_OE2 |
2046                                              GRC_LCLCTRL_GPIO_OUTPUT0 |
2047                                              GRC_LCLCTRL_GPIO_OUTPUT1 |
2048                                              tp->grc_local_ctrl;
2049                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2050
2051                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
2052                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2053
2054                         grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
2055                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2056                 } else {
2057                         u32 no_gpio2;
2058                         u32 grc_local_ctrl = 0;
2059
2060                         if (tp_peer != tp &&
2061                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
2062                                 return;
2063
2064                         /* Workaround to prevent overdrawing Amps. */
2065                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2066                             ASIC_REV_5714) {
2067                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
2068                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2069                                             grc_local_ctrl, 100);
2070                         }
2071
2072                         /* On 5753 and variants, GPIO2 cannot be used. */
2073                         no_gpio2 = tp->nic_sram_data_cfg &
2074                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
2075
2076                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
2077                                          GRC_LCLCTRL_GPIO_OE1 |
2078                                          GRC_LCLCTRL_GPIO_OE2 |
2079                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
2080                                          GRC_LCLCTRL_GPIO_OUTPUT2;
2081                         if (no_gpio2) {
2082                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
2083                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
2084                         }
2085                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2086                                                     grc_local_ctrl, 100);
2087
2088                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2089
2090                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2091                                                     grc_local_ctrl, 100);
2092
2093                         if (!no_gpio2) {
2094                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2095                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2096                                             grc_local_ctrl, 100);
2097                         }
2098                 }
2099         } else {
2100                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
2101                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
2102                         if (tp_peer != tp &&
2103                             (tp_peer->tg3_flags & TG3_FLAG_INIT_COMPLETE) != 0)
2104                                 return;
2105
2106                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2107                                     (GRC_LCLCTRL_GPIO_OE1 |
2108                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2109
2110                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2111                                     GRC_LCLCTRL_GPIO_OE1, 100);
2112
2113                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2114                                     (GRC_LCLCTRL_GPIO_OE1 |
2115                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2116                 }
2117         }
2118 }
2119
2120 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
2121 {
2122         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
2123                 return 1;
2124         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411) {
2125                 if (speed != SPEED_10)
2126                         return 1;
2127         } else if (speed == SPEED_10)
2128                 return 1;
2129
2130         return 0;
2131 }
2132
2133 static int tg3_setup_phy(struct tg3 *, int);
2134
2135 #define RESET_KIND_SHUTDOWN     0
2136 #define RESET_KIND_INIT         1
2137 #define RESET_KIND_SUSPEND      2
2138
2139 static void tg3_write_sig_post_reset(struct tg3 *, int);
2140 static int tg3_halt_cpu(struct tg3 *, u32);
2141
2142 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
2143 {
2144         u32 val;
2145
2146         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
2147                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2148                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
2149                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
2150
2151                         sg_dig_ctrl |=
2152                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
2153                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
2154                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
2155                 }
2156                 return;
2157         }
2158
2159         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2160                 tg3_bmcr_reset(tp);
2161                 val = tr32(GRC_MISC_CFG);
2162                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
2163                 udelay(40);
2164                 return;
2165         } else if (do_low_power) {
2166                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
2167                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
2168
2169                 tg3_writephy(tp, MII_TG3_AUX_CTRL,
2170                              MII_TG3_AUXCTL_SHDWSEL_PWRCTL |
2171                              MII_TG3_AUXCTL_PCTL_100TX_LPWR |
2172                              MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
2173                              MII_TG3_AUXCTL_PCTL_VREG_11V);
2174         }
2175
2176         /* The PHY should not be powered down on some chips because
2177          * of bugs.
2178          */
2179         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2180             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2181             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
2182              (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)))
2183                 return;
2184
2185         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
2186             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
2187                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2188                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2189                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
2190                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2191         }
2192
2193         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
2194 }
2195
2196 /* tp->lock is held. */
2197 static int tg3_nvram_lock(struct tg3 *tp)
2198 {
2199         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
2200                 int i;
2201
2202                 if (tp->nvram_lock_cnt == 0) {
2203                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
2204                         for (i = 0; i < 8000; i++) {
2205                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
2206                                         break;
2207                                 udelay(20);
2208                         }
2209                         if (i == 8000) {
2210                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
2211                                 return -ENODEV;
2212                         }
2213                 }
2214                 tp->nvram_lock_cnt++;
2215         }
2216         return 0;
2217 }
2218
2219 /* tp->lock is held. */
2220 static void tg3_nvram_unlock(struct tg3 *tp)
2221 {
2222         if (tp->tg3_flags & TG3_FLAG_NVRAM) {
2223                 if (tp->nvram_lock_cnt > 0)
2224                         tp->nvram_lock_cnt--;
2225                 if (tp->nvram_lock_cnt == 0)
2226                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
2227         }
2228 }
2229
2230 /* tp->lock is held. */
2231 static void tg3_enable_nvram_access(struct tg3 *tp)
2232 {
2233         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2234             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
2235                 u32 nvaccess = tr32(NVRAM_ACCESS);
2236
2237                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
2238         }
2239 }
2240
2241 /* tp->lock is held. */
2242 static void tg3_disable_nvram_access(struct tg3 *tp)
2243 {
2244         if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2245             !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) {
2246                 u32 nvaccess = tr32(NVRAM_ACCESS);
2247
2248                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
2249         }
2250 }
2251
2252 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
2253                                         u32 offset, u32 *val)
2254 {
2255         u32 tmp;
2256         int i;
2257
2258         if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
2259                 return -EINVAL;
2260
2261         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
2262                                         EEPROM_ADDR_DEVID_MASK |
2263                                         EEPROM_ADDR_READ);
2264         tw32(GRC_EEPROM_ADDR,
2265              tmp |
2266              (0 << EEPROM_ADDR_DEVID_SHIFT) |
2267              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
2268               EEPROM_ADDR_ADDR_MASK) |
2269              EEPROM_ADDR_READ | EEPROM_ADDR_START);
2270
2271         for (i = 0; i < 1000; i++) {
2272                 tmp = tr32(GRC_EEPROM_ADDR);
2273
2274                 if (tmp & EEPROM_ADDR_COMPLETE)
2275                         break;
2276                 msleep(1);
2277         }
2278         if (!(tmp & EEPROM_ADDR_COMPLETE))
2279                 return -EBUSY;
2280
2281         tmp = tr32(GRC_EEPROM_DATA);
2282
2283         /*
2284          * The data will always be opposite the native endian
2285          * format.  Perform a blind byteswap to compensate.
2286          */
2287         *val = swab32(tmp);
2288
2289         return 0;
2290 }
2291
2292 #define NVRAM_CMD_TIMEOUT 10000
2293
2294 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
2295 {
2296         int i;
2297
2298         tw32(NVRAM_CMD, nvram_cmd);
2299         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
2300                 udelay(10);
2301                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
2302                         udelay(10);
2303                         break;
2304                 }
2305         }
2306
2307         if (i == NVRAM_CMD_TIMEOUT)
2308                 return -EBUSY;
2309
2310         return 0;
2311 }
2312
2313 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
2314 {
2315         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
2316             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
2317             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
2318            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
2319             (tp->nvram_jedecnum == JEDEC_ATMEL))
2320
2321                 addr = ((addr / tp->nvram_pagesize) <<
2322                         ATMEL_AT45DB0X1B_PAGE_POS) +
2323                        (addr % tp->nvram_pagesize);
2324
2325         return addr;
2326 }
2327
2328 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
2329 {
2330         if ((tp->tg3_flags & TG3_FLAG_NVRAM) &&
2331             (tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) &&
2332             (tp->tg3_flags2 & TG3_FLG2_FLASH) &&
2333            !(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM_ADDR_TRANS) &&
2334             (tp->nvram_jedecnum == JEDEC_ATMEL))
2335
2336                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
2337                         tp->nvram_pagesize) +
2338                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
2339
2340         return addr;
2341 }
2342
2343 /* NOTE: Data read in from NVRAM is byteswapped according to
2344  * the byteswapping settings for all other register accesses.
2345  * tg3 devices are BE devices, so on a BE machine, the data
2346  * returned will be exactly as it is seen in NVRAM.  On a LE
2347  * machine, the 32-bit value will be byteswapped.
2348  */
2349 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
2350 {
2351         int ret;
2352
2353         if (!(tp->tg3_flags & TG3_FLAG_NVRAM))
2354                 return tg3_nvram_read_using_eeprom(tp, offset, val);
2355
2356         offset = tg3_nvram_phys_addr(tp, offset);
2357
2358         if (offset > NVRAM_ADDR_MSK)
2359                 return -EINVAL;
2360
2361         ret = tg3_nvram_lock(tp);
2362         if (ret)
2363                 return ret;
2364
2365         tg3_enable_nvram_access(tp);
2366
2367         tw32(NVRAM_ADDR, offset);
2368         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
2369                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
2370
2371         if (ret == 0)
2372                 *val = tr32(NVRAM_RDDATA);
2373
2374         tg3_disable_nvram_access(tp);
2375
2376         tg3_nvram_unlock(tp);
2377
2378         return ret;
2379 }
2380
2381 /* Ensures NVRAM data is in bytestream format. */
2382 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
2383 {
2384         u32 v;
2385         int res = tg3_nvram_read(tp, offset, &v);
2386         if (!res)
2387                 *val = cpu_to_be32(v);
2388         return res;
2389 }
2390
2391 /* tp->lock is held. */
2392 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
2393 {
2394         u32 addr_high, addr_low;
2395         int i;
2396
2397         addr_high = ((tp->dev->dev_addr[0] << 8) |
2398                      tp->dev->dev_addr[1]);
2399         addr_low = ((tp->dev->dev_addr[2] << 24) |
2400                     (tp->dev->dev_addr[3] << 16) |
2401                     (tp->dev->dev_addr[4] <<  8) |
2402                     (tp->dev->dev_addr[5] <<  0));
2403         for (i = 0; i < 4; i++) {
2404                 if (i == 1 && skip_mac_1)
2405                         continue;
2406                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
2407                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
2408         }
2409
2410         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2411             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2412                 for (i = 0; i < 12; i++) {
2413                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
2414                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
2415                 }
2416         }
2417
2418         addr_high = (tp->dev->dev_addr[0] +
2419                      tp->dev->dev_addr[1] +
2420                      tp->dev->dev_addr[2] +
2421                      tp->dev->dev_addr[3] +
2422                      tp->dev->dev_addr[4] +
2423                      tp->dev->dev_addr[5]) &
2424                 TX_BACKOFF_SEED_MASK;
2425         tw32(MAC_TX_BACKOFF_SEED, addr_high);
2426 }
2427
2428 static int tg3_set_power_state(struct tg3 *tp, pci_power_t state)
2429 {
2430         u32 misc_host_ctrl;
2431         bool device_should_wake, do_low_power;
2432
2433         /* Make sure register accesses (indirect or otherwise)
2434          * will function correctly.
2435          */
2436         pci_write_config_dword(tp->pdev,
2437                                TG3PCI_MISC_HOST_CTRL,
2438                                tp->misc_host_ctrl);
2439
2440         switch (state) {
2441         case PCI_D0:
2442                 pci_enable_wake(tp->pdev, state, false);
2443                 pci_set_power_state(tp->pdev, PCI_D0);
2444
2445                 /* Switch out of Vaux if it is a NIC */
2446                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
2447                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
2448
2449                 return 0;
2450
2451         case PCI_D1:
2452         case PCI_D2:
2453         case PCI_D3hot:
2454                 break;
2455
2456         default:
2457                 printk(KERN_ERR PFX "%s: Invalid power state (D%d) requested\n",
2458                         tp->dev->name, state);
2459                 return -EINVAL;
2460         }
2461
2462         /* Restore the CLKREQ setting. */
2463         if (tp->tg3_flags3 & TG3_FLG3_CLKREQ_BUG) {
2464                 u16 lnkctl;
2465
2466                 pci_read_config_word(tp->pdev,
2467                                      tp->pcie_cap + PCI_EXP_LNKCTL,
2468                                      &lnkctl);
2469                 lnkctl |= PCI_EXP_LNKCTL_CLKREQ_EN;
2470                 pci_write_config_word(tp->pdev,
2471                                       tp->pcie_cap + PCI_EXP_LNKCTL,
2472                                       lnkctl);
2473         }
2474
2475         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
2476         tw32(TG3PCI_MISC_HOST_CTRL,
2477              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
2478
2479         device_should_wake = pci_pme_capable(tp->pdev, state) &&
2480                              device_may_wakeup(&tp->pdev->dev) &&
2481                              (tp->tg3_flags & TG3_FLAG_WOL_ENABLE);
2482
2483         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
2484                 do_low_power = false;
2485                 if ((tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) &&
2486                     !tp->link_config.phy_is_low_power) {
2487                         struct phy_device *phydev;
2488                         u32 phyid, advertising;
2489
2490                         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
2491
2492                         tp->link_config.phy_is_low_power = 1;
2493
2494                         tp->link_config.orig_speed = phydev->speed;
2495                         tp->link_config.orig_duplex = phydev->duplex;
2496                         tp->link_config.orig_autoneg = phydev->autoneg;
2497                         tp->link_config.orig_advertising = phydev->advertising;
2498
2499                         advertising = ADVERTISED_TP |
2500                                       ADVERTISED_Pause |
2501                                       ADVERTISED_Autoneg |
2502                                       ADVERTISED_10baseT_Half;
2503
2504                         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
2505                             device_should_wake) {
2506                                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
2507                                         advertising |=
2508                                                 ADVERTISED_100baseT_Half |
2509                                                 ADVERTISED_100baseT_Full |
2510                                                 ADVERTISED_10baseT_Full;
2511                                 else
2512                                         advertising |= ADVERTISED_10baseT_Full;
2513                         }
2514
2515                         phydev->advertising = advertising;
2516
2517                         phy_start_aneg(phydev);
2518
2519                         phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
2520                         if (phyid != TG3_PHY_ID_BCMAC131) {
2521                                 phyid &= TG3_PHY_OUI_MASK;
2522                                 if (phyid == TG3_PHY_OUI_1 ||
2523                                     phyid == TG3_PHY_OUI_2 ||
2524                                     phyid == TG3_PHY_OUI_3)
2525                                         do_low_power = true;
2526                         }
2527                 }
2528         } else {
2529                 do_low_power = true;
2530
2531                 if (tp->link_config.phy_is_low_power == 0) {
2532                         tp->link_config.phy_is_low_power = 1;
2533                         tp->link_config.orig_speed = tp->link_config.speed;
2534                         tp->link_config.orig_duplex = tp->link_config.duplex;
2535                         tp->link_config.orig_autoneg = tp->link_config.autoneg;
2536                 }
2537
2538                 if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
2539                         tp->link_config.speed = SPEED_10;
2540                         tp->link_config.duplex = DUPLEX_HALF;
2541                         tp->link_config.autoneg = AUTONEG_ENABLE;
2542                         tg3_setup_phy(tp, 0);
2543                 }
2544         }
2545
2546         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2547                 u32 val;
2548
2549                 val = tr32(GRC_VCPU_EXT_CTRL);
2550                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
2551         } else if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
2552                 int i;
2553                 u32 val;
2554
2555                 for (i = 0; i < 200; i++) {
2556                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
2557                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
2558                                 break;
2559                         msleep(1);
2560                 }
2561         }
2562         if (tp->tg3_flags & TG3_FLAG_WOL_CAP)
2563                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
2564                                                      WOL_DRV_STATE_SHUTDOWN |
2565                                                      WOL_DRV_WOL |
2566                                                      WOL_SET_MAGIC_PKT);
2567
2568         if (device_should_wake) {
2569                 u32 mac_mode;
2570
2571                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
2572                         if (do_low_power) {
2573                                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
2574                                 udelay(40);
2575                         }
2576
2577                         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
2578                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
2579                         else
2580                                 mac_mode = MAC_MODE_PORT_MODE_MII;
2581
2582                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
2583                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2584                             ASIC_REV_5700) {
2585                                 u32 speed = (tp->tg3_flags &
2586                                              TG3_FLAG_WOL_SPEED_100MB) ?
2587                                              SPEED_100 : SPEED_10;
2588                                 if (tg3_5700_link_polarity(tp, speed))
2589                                         mac_mode |= MAC_MODE_LINK_POLARITY;
2590                                 else
2591                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
2592                         }
2593                 } else {
2594                         mac_mode = MAC_MODE_PORT_MODE_TBI;
2595                 }
2596
2597                 if (!(tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
2598                         tw32(MAC_LED_CTRL, tp->led_ctrl);
2599
2600                 mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
2601                 if (((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
2602                     !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) &&
2603                     ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
2604                      (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)))
2605                         mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
2606
2607                 if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
2608                         mac_mode |= tp->mac_mode &
2609                                     (MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN);
2610                         if (mac_mode & MAC_MODE_APE_TX_EN)
2611                                 mac_mode |= MAC_MODE_TDE_ENABLE;
2612                 }
2613
2614                 tw32_f(MAC_MODE, mac_mode);
2615                 udelay(100);
2616
2617                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
2618                 udelay(10);
2619         }
2620
2621         if (!(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) &&
2622             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2623              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
2624                 u32 base_val;
2625
2626                 base_val = tp->pci_clock_ctrl;
2627                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
2628                              CLOCK_CTRL_TXCLK_DISABLE);
2629
2630                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
2631                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
2632         } else if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
2633                    (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
2634                    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)) {
2635                 /* do nothing */
2636         } else if (!((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
2637                      (tp->tg3_flags & TG3_FLAG_ENABLE_ASF))) {
2638                 u32 newbits1, newbits2;
2639
2640                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2641                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2642                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
2643                                     CLOCK_CTRL_TXCLK_DISABLE |
2644                                     CLOCK_CTRL_ALTCLK);
2645                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2646                 } else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
2647                         newbits1 = CLOCK_CTRL_625_CORE;
2648                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
2649                 } else {
2650                         newbits1 = CLOCK_CTRL_ALTCLK;
2651                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2652                 }
2653
2654                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
2655                             40);
2656
2657                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
2658                             40);
2659
2660                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
2661                         u32 newbits3;
2662
2663                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2664                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2665                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
2666                                             CLOCK_CTRL_TXCLK_DISABLE |
2667                                             CLOCK_CTRL_44MHZ_CORE);
2668                         } else {
2669                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
2670                         }
2671
2672                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
2673                                     tp->pci_clock_ctrl | newbits3, 40);
2674                 }
2675         }
2676
2677         if (!(device_should_wake) &&
2678             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
2679                 tg3_power_down_phy(tp, do_low_power);
2680
2681         tg3_frob_aux_power(tp);
2682
2683         /* Workaround for unstable PLL clock */
2684         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
2685             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
2686                 u32 val = tr32(0x7d00);
2687
2688                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
2689                 tw32(0x7d00, val);
2690                 if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
2691                         int err;
2692
2693                         err = tg3_nvram_lock(tp);
2694                         tg3_halt_cpu(tp, RX_CPU_BASE);
2695                         if (!err)
2696                                 tg3_nvram_unlock(tp);
2697                 }
2698         }
2699
2700         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
2701
2702         if (device_should_wake)
2703                 pci_enable_wake(tp->pdev, state, true);
2704
2705         /* Finally, set the new power state. */
2706         pci_set_power_state(tp->pdev, state);
2707
2708         return 0;
2709 }
2710
2711 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
2712 {
2713         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
2714         case MII_TG3_AUX_STAT_10HALF:
2715                 *speed = SPEED_10;
2716                 *duplex = DUPLEX_HALF;
2717                 break;
2718
2719         case MII_TG3_AUX_STAT_10FULL:
2720                 *speed = SPEED_10;
2721                 *duplex = DUPLEX_FULL;
2722                 break;
2723
2724         case MII_TG3_AUX_STAT_100HALF:
2725                 *speed = SPEED_100;
2726                 *duplex = DUPLEX_HALF;
2727                 break;
2728
2729         case MII_TG3_AUX_STAT_100FULL:
2730                 *speed = SPEED_100;
2731                 *duplex = DUPLEX_FULL;
2732                 break;
2733
2734         case MII_TG3_AUX_STAT_1000HALF:
2735                 *speed = SPEED_1000;
2736                 *duplex = DUPLEX_HALF;
2737                 break;
2738
2739         case MII_TG3_AUX_STAT_1000FULL:
2740                 *speed = SPEED_1000;
2741                 *duplex = DUPLEX_FULL;
2742                 break;
2743
2744         default:
2745                 if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
2746                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
2747                                  SPEED_10;
2748                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
2749                                   DUPLEX_HALF;
2750                         break;
2751                 }
2752                 *speed = SPEED_INVALID;
2753                 *duplex = DUPLEX_INVALID;
2754                 break;
2755         }
2756 }
2757
2758 static void tg3_phy_copper_begin(struct tg3 *tp)
2759 {
2760         u32 new_adv;
2761         int i;
2762
2763         if (tp->link_config.phy_is_low_power) {
2764                 /* Entering low power mode.  Disable gigabit and
2765                  * 100baseT advertisements.
2766                  */
2767                 tg3_writephy(tp, MII_TG3_CTRL, 0);
2768
2769                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
2770                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
2771                 if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
2772                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
2773
2774                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2775         } else if (tp->link_config.speed == SPEED_INVALID) {
2776                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
2777                         tp->link_config.advertising &=
2778                                 ~(ADVERTISED_1000baseT_Half |
2779                                   ADVERTISED_1000baseT_Full);
2780
2781                 new_adv = ADVERTISE_CSMA;
2782                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
2783                         new_adv |= ADVERTISE_10HALF;
2784                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
2785                         new_adv |= ADVERTISE_10FULL;
2786                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
2787                         new_adv |= ADVERTISE_100HALF;
2788                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
2789                         new_adv |= ADVERTISE_100FULL;
2790
2791                 new_adv |= tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2792
2793                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2794
2795                 if (tp->link_config.advertising &
2796                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
2797                         new_adv = 0;
2798                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2799                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
2800                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2801                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
2802                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
2803                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2804                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
2805                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2806                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2807                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
2808                 } else {
2809                         tg3_writephy(tp, MII_TG3_CTRL, 0);
2810                 }
2811         } else {
2812                 new_adv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2813                 new_adv |= ADVERTISE_CSMA;
2814
2815                 /* Asking for a specific link mode. */
2816                 if (tp->link_config.speed == SPEED_1000) {
2817                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2818
2819                         if (tp->link_config.duplex == DUPLEX_FULL)
2820                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
2821                         else
2822                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
2823                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2824                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
2825                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2826                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2827                 } else {
2828                         if (tp->link_config.speed == SPEED_100) {
2829                                 if (tp->link_config.duplex == DUPLEX_FULL)
2830                                         new_adv |= ADVERTISE_100FULL;
2831                                 else
2832                                         new_adv |= ADVERTISE_100HALF;
2833                         } else {
2834                                 if (tp->link_config.duplex == DUPLEX_FULL)
2835                                         new_adv |= ADVERTISE_10FULL;
2836                                 else
2837                                         new_adv |= ADVERTISE_10HALF;
2838                         }
2839                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2840
2841                         new_adv = 0;
2842                 }
2843
2844                 tg3_writephy(tp, MII_TG3_CTRL, new_adv);
2845         }
2846
2847         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
2848             tp->link_config.speed != SPEED_INVALID) {
2849                 u32 bmcr, orig_bmcr;
2850
2851                 tp->link_config.active_speed = tp->link_config.speed;
2852                 tp->link_config.active_duplex = tp->link_config.duplex;
2853
2854                 bmcr = 0;
2855                 switch (tp->link_config.speed) {
2856                 default:
2857                 case SPEED_10:
2858                         break;
2859
2860                 case SPEED_100:
2861                         bmcr |= BMCR_SPEED100;
2862                         break;
2863
2864                 case SPEED_1000:
2865                         bmcr |= TG3_BMCR_SPEED1000;
2866                         break;
2867                 }
2868
2869                 if (tp->link_config.duplex == DUPLEX_FULL)
2870                         bmcr |= BMCR_FULLDPLX;
2871
2872                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
2873                     (bmcr != orig_bmcr)) {
2874                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
2875                         for (i = 0; i < 1500; i++) {
2876                                 u32 tmp;
2877
2878                                 udelay(10);
2879                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
2880                                     tg3_readphy(tp, MII_BMSR, &tmp))
2881                                         continue;
2882                                 if (!(tmp & BMSR_LSTATUS)) {
2883                                         udelay(40);
2884                                         break;
2885                                 }
2886                         }
2887                         tg3_writephy(tp, MII_BMCR, bmcr);
2888                         udelay(40);
2889                 }
2890         } else {
2891                 tg3_writephy(tp, MII_BMCR,
2892                              BMCR_ANENABLE | BMCR_ANRESTART);
2893         }
2894 }
2895
2896 static int tg3_init_5401phy_dsp(struct tg3 *tp)
2897 {
2898         int err;
2899
2900         /* Turn off tap power management. */
2901         /* Set Extended packet length bit */
2902         err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4c20);
2903
2904         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
2905         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);
2906
2907         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
2908         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);
2909
2910         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2911         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);
2912
2913         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
2914         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);
2915
2916         err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
2917         err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);
2918
2919         udelay(40);
2920
2921         return err;
2922 }
2923
2924 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
2925 {
2926         u32 adv_reg, all_mask = 0;
2927
2928         if (mask & ADVERTISED_10baseT_Half)
2929                 all_mask |= ADVERTISE_10HALF;
2930         if (mask & ADVERTISED_10baseT_Full)
2931                 all_mask |= ADVERTISE_10FULL;
2932         if (mask & ADVERTISED_100baseT_Half)
2933                 all_mask |= ADVERTISE_100HALF;
2934         if (mask & ADVERTISED_100baseT_Full)
2935                 all_mask |= ADVERTISE_100FULL;
2936
2937         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
2938                 return 0;
2939
2940         if ((adv_reg & all_mask) != all_mask)
2941                 return 0;
2942         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
2943                 u32 tg3_ctrl;
2944
2945                 all_mask = 0;
2946                 if (mask & ADVERTISED_1000baseT_Half)
2947                         all_mask |= ADVERTISE_1000HALF;
2948                 if (mask & ADVERTISED_1000baseT_Full)
2949                         all_mask |= ADVERTISE_1000FULL;
2950
2951                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
2952                         return 0;
2953
2954                 if ((tg3_ctrl & all_mask) != all_mask)
2955                         return 0;
2956         }
2957         return 1;
2958 }
2959
2960 static int tg3_adv_1000T_flowctrl_ok(struct tg3 *tp, u32 *lcladv, u32 *rmtadv)
2961 {
2962         u32 curadv, reqadv;
2963
2964         if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
2965                 return 1;
2966
2967         curadv = *lcladv & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
2968         reqadv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2969
2970         if (tp->link_config.active_duplex == DUPLEX_FULL) {
2971                 if (curadv != reqadv)
2972                         return 0;
2973
2974                 if (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG)
2975                         tg3_readphy(tp, MII_LPA, rmtadv);
2976         } else {
2977                 /* Reprogram the advertisement register, even if it
2978                  * does not affect the current link.  If the link
2979                  * gets renegotiated in the future, we can save an
2980                  * additional renegotiation cycle by advertising
2981                  * it correctly in the first place.
2982                  */
2983                 if (curadv != reqadv) {
2984                         *lcladv &= ~(ADVERTISE_PAUSE_CAP |
2985                                      ADVERTISE_PAUSE_ASYM);
2986                         tg3_writephy(tp, MII_ADVERTISE, *lcladv | reqadv);
2987                 }
2988         }
2989
2990         return 1;
2991 }
2992
2993 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
2994 {
2995         int current_link_up;
2996         u32 bmsr, dummy;
2997         u32 lcl_adv, rmt_adv;
2998         u16 current_speed;
2999         u8 current_duplex;
3000         int i, err;
3001
3002         tw32(MAC_EVENT, 0);
3003
3004         tw32_f(MAC_STATUS,
3005              (MAC_STATUS_SYNC_CHANGED |
3006               MAC_STATUS_CFG_CHANGED |
3007               MAC_STATUS_MI_COMPLETION |
3008               MAC_STATUS_LNKSTATE_CHANGED));
3009         udelay(40);
3010
3011         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
3012                 tw32_f(MAC_MI_MODE,
3013                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
3014                 udelay(80);
3015         }
3016
3017         tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);
3018
3019         /* Some third-party PHYs need to be reset on link going
3020          * down.
3021          */
3022         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
3023              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
3024              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
3025             netif_carrier_ok(tp->dev)) {
3026                 tg3_readphy(tp, MII_BMSR, &bmsr);
3027                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3028                     !(bmsr & BMSR_LSTATUS))
3029                         force_reset = 1;
3030         }
3031         if (force_reset)
3032                 tg3_phy_reset(tp);
3033
3034         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
3035                 tg3_readphy(tp, MII_BMSR, &bmsr);
3036                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
3037                     !(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
3038                         bmsr = 0;
3039
3040                 if (!(bmsr & BMSR_LSTATUS)) {
3041                         err = tg3_init_5401phy_dsp(tp);
3042                         if (err)
3043                                 return err;
3044
3045                         tg3_readphy(tp, MII_BMSR, &bmsr);
3046                         for (i = 0; i < 1000; i++) {
3047                                 udelay(10);
3048                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3049                                     (bmsr & BMSR_LSTATUS)) {
3050                                         udelay(40);
3051                                         break;
3052                                 }
3053                         }
3054
3055                         if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
3056                             !(bmsr & BMSR_LSTATUS) &&
3057                             tp->link_config.active_speed == SPEED_1000) {
3058                                 err = tg3_phy_reset(tp);
3059                                 if (!err)
3060                                         err = tg3_init_5401phy_dsp(tp);
3061                                 if (err)
3062                                         return err;
3063                         }
3064                 }
3065         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
3066                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
3067                 /* 5701 {A0,B0} CRC bug workaround */
3068                 tg3_writephy(tp, 0x15, 0x0a75);
3069                 tg3_writephy(tp, 0x1c, 0x8c68);
3070                 tg3_writephy(tp, 0x1c, 0x8d68);
3071                 tg3_writephy(tp, 0x1c, 0x8c68);
3072         }
3073
3074         /* Clear pending interrupts... */
3075         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
3076         tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
3077
3078         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
3079                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
3080         else if (!(tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET))
3081                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
3082
3083         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
3084             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
3085                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
3086                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
3087                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
3088                 else
3089                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
3090         }
3091
3092         current_link_up = 0;
3093         current_speed = SPEED_INVALID;
3094         current_duplex = DUPLEX_INVALID;
3095
3096         if (tp->tg3_flags2 & TG3_FLG2_CAPACITIVE_COUPLING) {
3097                 u32 val;
3098
3099                 tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x4007);
3100                 tg3_readphy(tp, MII_TG3_AUX_CTRL, &val);
3101                 if (!(val & (1 << 10))) {
3102                         val |= (1 << 10);
3103                         tg3_writephy(tp, MII_TG3_AUX_CTRL, val);
3104                         goto relink;
3105                 }
3106         }
3107
3108         bmsr = 0;
3109         for (i = 0; i < 100; i++) {
3110                 tg3_readphy(tp, MII_BMSR, &bmsr);
3111                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3112                     (bmsr & BMSR_LSTATUS))
3113                         break;
3114                 udelay(40);
3115         }
3116
3117         if (bmsr & BMSR_LSTATUS) {
3118                 u32 aux_stat, bmcr;
3119
3120                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
3121                 for (i = 0; i < 2000; i++) {
3122                         udelay(10);
3123                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
3124                             aux_stat)
3125                                 break;
3126                 }
3127
3128                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
3129                                              &current_speed,
3130                                              &current_duplex);
3131
3132                 bmcr = 0;
3133                 for (i = 0; i < 200; i++) {
3134                         tg3_readphy(tp, MII_BMCR, &bmcr);
3135                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
3136                                 continue;
3137                         if (bmcr && bmcr != 0x7fff)
3138                                 break;
3139                         udelay(10);
3140                 }
3141
3142                 lcl_adv = 0;
3143                 rmt_adv = 0;
3144
3145                 tp->link_config.active_speed = current_speed;
3146                 tp->link_config.active_duplex = current_duplex;
3147
3148                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3149                         if ((bmcr & BMCR_ANENABLE) &&
3150                             tg3_copper_is_advertising_all(tp,
3151                                                 tp->link_config.advertising)) {
3152                                 if (tg3_adv_1000T_flowctrl_ok(tp, &lcl_adv,
3153                                                                   &rmt_adv))
3154                                         current_link_up = 1;
3155                         }
3156                 } else {
3157                         if (!(bmcr & BMCR_ANENABLE) &&
3158                             tp->link_config.speed == current_speed &&
3159                             tp->link_config.duplex == current_duplex &&
3160                             tp->link_config.flowctrl ==
3161                             tp->link_config.active_flowctrl) {
3162                                 current_link_up = 1;
3163                         }
3164                 }
3165
3166                 if (current_link_up == 1 &&
3167                     tp->link_config.active_duplex == DUPLEX_FULL)
3168                         tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
3169         }
3170
3171 relink:
3172         if (current_link_up == 0 || tp->link_config.phy_is_low_power) {
3173                 u32 tmp;
3174
3175                 tg3_phy_copper_begin(tp);
3176
3177                 tg3_readphy(tp, MII_BMSR, &tmp);
3178                 if (!tg3_readphy(tp, MII_BMSR, &tmp) &&
3179                     (tmp & BMSR_LSTATUS))
3180                         current_link_up = 1;
3181         }
3182
3183         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
3184         if (current_link_up == 1) {
3185                 if (tp->link_config.active_speed == SPEED_100 ||
3186                     tp->link_config.active_speed == SPEED_10)
3187                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3188                 else
3189                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3190         } else if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET)
3191                 tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3192         else
3193                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3194
3195         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3196         if (tp->link_config.active_duplex == DUPLEX_HALF)
3197                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3198
3199         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
3200                 if (current_link_up == 1 &&
3201                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
3202                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
3203                 else
3204                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
3205         }
3206
3207         /* ??? Without this setting Netgear GA302T PHY does not
3208          * ??? send/receive packets...
3209          */
3210         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
3211             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
3212                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
3213                 tw32_f(MAC_MI_MODE, tp->mi_mode);
3214                 udelay(80);
3215         }
3216
3217         tw32_f(MAC_MODE, tp->mac_mode);
3218         udelay(40);
3219
3220         if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
3221                 /* Polled via timer. */
3222                 tw32_f(MAC_EVENT, 0);
3223         } else {
3224                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3225         }
3226         udelay(40);
3227
3228         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
3229             current_link_up == 1 &&
3230             tp->link_config.active_speed == SPEED_1000 &&
3231             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
3232              (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
3233                 udelay(120);
3234                 tw32_f(MAC_STATUS,
3235                      (MAC_STATUS_SYNC_CHANGED |
3236                       MAC_STATUS_CFG_CHANGED));
3237                 udelay(40);
3238                 tg3_write_mem(tp,
3239                               NIC_SRAM_FIRMWARE_MBOX,
3240                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
3241         }
3242
3243         /* Prevent send BD corruption. */
3244         if (tp->tg3_flags3 & TG3_FLG3_CLKREQ_BUG) {
3245                 u16 oldlnkctl, newlnkctl;
3246
3247                 pci_read_config_word(tp->pdev,
3248                                      tp->pcie_cap + PCI_EXP_LNKCTL,
3249                                      &oldlnkctl);
3250                 if (tp->link_config.active_speed == SPEED_100 ||
3251                     tp->link_config.active_speed == SPEED_10)
3252                         newlnkctl = oldlnkctl & ~PCI_EXP_LNKCTL_CLKREQ_EN;
3253                 else
3254                         newlnkctl = oldlnkctl | PCI_EXP_LNKCTL_CLKREQ_EN;
3255                 if (newlnkctl != oldlnkctl)
3256                         pci_write_config_word(tp->pdev,
3257                                               tp->pcie_cap + PCI_EXP_LNKCTL,
3258                                               newlnkctl);
3259         } else if (tp->tg3_flags3 & TG3_FLG3_TOGGLE_10_100_L1PLLPD) {
3260                 u32 newreg, oldreg = tr32(TG3_PCIE_LNKCTL);
3261                 if (tp->link_config.active_speed == SPEED_100 ||
3262                     tp->link_config.active_speed == SPEED_10)
3263                         newreg = oldreg & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
3264                 else
3265                         newreg = oldreg | TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
3266                 if (newreg != oldreg)
3267                         tw32(TG3_PCIE_LNKCTL, newreg);
3268         }
3269
3270         if (current_link_up != netif_carrier_ok(tp->dev)) {
3271                 if (current_link_up)
3272                         netif_carrier_on(tp->dev);
3273                 else
3274                         netif_carrier_off(tp->dev);
3275                 tg3_link_report(tp);
3276         }
3277
3278         return 0;
3279 }
3280
3281 struct tg3_fiber_aneginfo {
3282         int state;
3283 #define ANEG_STATE_UNKNOWN              0
3284 #define ANEG_STATE_AN_ENABLE            1
3285 #define ANEG_STATE_RESTART_INIT         2
3286 #define ANEG_STATE_RESTART              3
3287 #define ANEG_STATE_DISABLE_LINK_OK      4
3288 #define ANEG_STATE_ABILITY_DETECT_INIT  5
3289 #define ANEG_STATE_ABILITY_DETECT       6
3290 #define ANEG_STATE_ACK_DETECT_INIT      7
3291 #define ANEG_STATE_ACK_DETECT           8
3292 #define ANEG_STATE_COMPLETE_ACK_INIT    9
3293 #define ANEG_STATE_COMPLETE_ACK         10
3294 #define ANEG_STATE_IDLE_DETECT_INIT     11
3295 #define ANEG_STATE_IDLE_DETECT          12
3296 #define ANEG_STATE_LINK_OK              13
3297 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
3298 #define ANEG_STATE_NEXT_PAGE_WAIT       15
3299
3300         u32 flags;
3301 #define MR_AN_ENABLE            0x00000001
3302 #define MR_RESTART_AN           0x00000002
3303 #define MR_AN_COMPLETE          0x00000004
3304 #define MR_PAGE_RX              0x00000008
3305 #define MR_NP_LOADED            0x00000010
3306 #define MR_TOGGLE_TX            0x00000020
3307 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
3308 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
3309 #define MR_LP_ADV_SYM_PAUSE     0x00000100
3310 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
3311 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
3312 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
3313 #define MR_LP_ADV_NEXT_PAGE     0x00001000
3314 #define MR_TOGGLE_RX            0x00002000
3315 #define MR_NP_RX                0x00004000
3316
3317 #define MR_LINK_OK              0x80000000
3318
3319         unsigned long link_time, cur_time;
3320
3321         u32 ability_match_cfg;
3322         int ability_match_count;
3323
3324         char ability_match, idle_match, ack_match;
3325
3326         u32 txconfig, rxconfig;
3327 #define ANEG_CFG_NP             0x00000080
3328 #define ANEG_CFG_ACK            0x00000040
3329 #define ANEG_CFG_RF2            0x00000020
3330 #define ANEG_CFG_RF1            0x00000010
3331 #define ANEG_CFG_PS2            0x00000001
3332 #define ANEG_CFG_PS1            0x00008000
3333 #define ANEG_CFG_HD             0x00004000
3334 #define ANEG_CFG_FD             0x00002000
3335 #define ANEG_CFG_INVAL          0x00001f06
3336
3337 };
3338 #define ANEG_OK         0
3339 #define ANEG_DONE       1
3340 #define ANEG_TIMER_ENAB 2
3341 #define ANEG_FAILED     -1
3342
3343 #define ANEG_STATE_SETTLE_TIME  10000
3344
3345 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
3346                                    struct tg3_fiber_aneginfo *ap)
3347 {
3348         u16 flowctrl;
3349         unsigned long delta;
3350         u32 rx_cfg_reg;
3351         int ret;
3352
3353         if (ap->state == ANEG_STATE_UNKNOWN) {
3354                 ap->rxconfig = 0;
3355                 ap->link_time = 0;
3356                 ap->cur_time = 0;
3357                 ap->ability_match_cfg = 0;
3358                 ap->ability_match_count = 0;
3359                 ap->ability_match = 0;
3360                 ap->idle_match = 0;
3361                 ap->ack_match = 0;
3362         }
3363         ap->cur_time++;
3364
3365         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
3366                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
3367
3368                 if (rx_cfg_reg != ap->ability_match_cfg) {
3369                         ap->ability_match_cfg = rx_cfg_reg;
3370                         ap->ability_match = 0;
3371                         ap->ability_match_count = 0;
3372                 } else {
3373                         if (++ap->ability_match_count > 1) {
3374                                 ap->ability_match = 1;
3375                                 ap->ability_match_cfg = rx_cfg_reg;
3376                         }
3377                 }
3378                 if (rx_cfg_reg & ANEG_CFG_ACK)
3379                         ap->ack_match = 1;
3380                 else
3381                         ap->ack_match = 0;
3382
3383                 ap->idle_match = 0;
3384         } else {
3385                 ap->idle_match = 1;
3386                 ap->ability_match_cfg = 0;
3387                 ap->ability_match_count = 0;
3388                 ap->ability_match = 0;
3389                 ap->ack_match = 0;
3390
3391                 rx_cfg_reg = 0;
3392         }
3393
3394         ap->rxconfig = rx_cfg_reg;
3395         ret = ANEG_OK;
3396
3397         switch(ap->state) {
3398         case ANEG_STATE_UNKNOWN:
3399                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
3400                         ap->state = ANEG_STATE_AN_ENABLE;
3401
3402                 /* fallthru */
3403         case ANEG_STATE_AN_ENABLE:
3404                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
3405                 if (ap->flags & MR_AN_ENABLE) {
3406                         ap->link_time = 0;
3407                         ap->cur_time = 0;
3408                         ap->ability_match_cfg = 0;
3409                         ap->ability_match_count = 0;
3410                         ap->ability_match = 0;
3411                         ap->idle_match = 0;
3412                         ap->ack_match = 0;
3413
3414                         ap->state = ANEG_STATE_RESTART_INIT;
3415                 } else {
3416                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
3417                 }
3418                 break;
3419
3420         case ANEG_STATE_RESTART_INIT:
3421                 ap->link_time = ap->cur_time;
3422                 ap->flags &= ~(MR_NP_LOADED);
3423                 ap->txconfig = 0;
3424                 tw32(MAC_TX_AUTO_NEG, 0);
3425                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3426                 tw32_f(MAC_MODE, tp->mac_mode);
3427                 udelay(40);
3428
3429                 ret = ANEG_TIMER_ENAB;
3430                 ap->state = ANEG_STATE_RESTART;
3431
3432                 /* fallthru */
3433         case ANEG_STATE_RESTART:
3434                 delta = ap->cur_time - ap->link_time;
3435                 if (delta > ANEG_STATE_SETTLE_TIME) {
3436                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
3437                 } else {
3438                         ret = ANEG_TIMER_ENAB;
3439                 }
3440                 break;
3441
3442         case ANEG_STATE_DISABLE_LINK_OK:
3443                 ret = ANEG_DONE;
3444                 break;
3445
3446         case ANEG_STATE_ABILITY_DETECT_INIT:
3447                 ap->flags &= ~(MR_TOGGLE_TX);
3448                 ap->txconfig = ANEG_CFG_FD;
3449                 flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3450                 if (flowctrl & ADVERTISE_1000XPAUSE)
3451                         ap->txconfig |= ANEG_CFG_PS1;
3452                 if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3453                         ap->txconfig |= ANEG_CFG_PS2;
3454                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3455                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3456                 tw32_f(MAC_MODE, tp->mac_mode);
3457                 udelay(40);
3458
3459                 ap->state = ANEG_STATE_ABILITY_DETECT;
3460                 break;
3461
3462         case ANEG_STATE_ABILITY_DETECT:
3463                 if (ap->ability_match != 0 && ap->rxconfig != 0) {
3464                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
3465                 }
3466                 break;
3467
3468         case ANEG_STATE_ACK_DETECT_INIT:
3469                 ap->txconfig |= ANEG_CFG_ACK;
3470                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3471                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3472                 tw32_f(MAC_MODE, tp->mac_mode);
3473                 udelay(40);
3474
3475                 ap->state = ANEG_STATE_ACK_DETECT;
3476
3477                 /* fallthru */
3478         case ANEG_STATE_ACK_DETECT:
3479                 if (ap->ack_match != 0) {
3480                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
3481                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
3482                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
3483                         } else {
3484                                 ap->state = ANEG_STATE_AN_ENABLE;
3485                         }
3486                 } else if (ap->ability_match != 0 &&
3487                            ap->rxconfig == 0) {
3488                         ap->state = ANEG_STATE_AN_ENABLE;
3489                 }
3490                 break;
3491
3492         case ANEG_STATE_COMPLETE_ACK_INIT:
3493                 if (ap->rxconfig & ANEG_CFG_INVAL) {
3494                         ret = ANEG_FAILED;
3495                         break;
3496                 }
3497                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
3498                                MR_LP_ADV_HALF_DUPLEX |
3499                                MR_LP_ADV_SYM_PAUSE |
3500                                MR_LP_ADV_ASYM_PAUSE |
3501                                MR_LP_ADV_REMOTE_FAULT1 |
3502                                MR_LP_ADV_REMOTE_FAULT2 |
3503                                MR_LP_ADV_NEXT_PAGE |
3504                                MR_TOGGLE_RX |
3505                                MR_NP_RX);
3506                 if (ap->rxconfig & ANEG_CFG_FD)
3507                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
3508                 if (ap->rxconfig & ANEG_CFG_HD)
3509                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
3510                 if (ap->rxconfig & ANEG_CFG_PS1)
3511                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
3512                 if (ap->rxconfig & ANEG_CFG_PS2)
3513                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
3514                 if (ap->rxconfig & ANEG_CFG_RF1)
3515                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
3516                 if (ap->rxconfig & ANEG_CFG_RF2)
3517                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
3518                 if (ap->rxconfig & ANEG_CFG_NP)
3519                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
3520
3521                 ap->link_time = ap->cur_time;
3522
3523                 ap->flags ^= (MR_TOGGLE_TX);
3524                 if (ap->rxconfig & 0x0008)
3525                         ap->flags |= MR_TOGGLE_RX;
3526                 if (ap->rxconfig & ANEG_CFG_NP)
3527                         ap->flags |= MR_NP_RX;
3528                 ap->flags |= MR_PAGE_RX;
3529
3530                 ap->state = ANEG_STATE_COMPLETE_ACK;
3531                 ret = ANEG_TIMER_ENAB;
3532                 break;
3533
3534         case ANEG_STATE_COMPLETE_ACK:
3535                 if (ap->ability_match != 0 &&
3536                     ap->rxconfig == 0) {
3537                         ap->state = ANEG_STATE_AN_ENABLE;
3538                         break;
3539                 }
3540                 delta = ap->cur_time - ap->link_time;
3541                 if (delta > ANEG_STATE_SETTLE_TIME) {
3542                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
3543                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3544                         } else {
3545                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
3546                                     !(ap->flags & MR_NP_RX)) {
3547                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3548                                 } else {
3549                                         ret = ANEG_FAILED;
3550                                 }
3551                         }
3552                 }
3553                 break;
3554
3555         case ANEG_STATE_IDLE_DETECT_INIT:
3556                 ap->link_time = ap->cur_time;
3557                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3558                 tw32_f(MAC_MODE, tp->mac_mode);
3559                 udelay(40);
3560
3561                 ap->state = ANEG_STATE_IDLE_DETECT;
3562                 ret = ANEG_TIMER_ENAB;
3563                 break;
3564
3565         case ANEG_STATE_IDLE_DETECT:
3566                 if (ap->ability_match != 0 &&
3567                     ap->rxconfig == 0) {
3568                         ap->state = ANEG_STATE_AN_ENABLE;
3569                         break;
3570                 }
3571                 delta = ap->cur_time - ap->link_time;
3572                 if (delta > ANEG_STATE_SETTLE_TIME) {
3573                         /* XXX another gem from the Broadcom driver :( */
3574                         ap->state = ANEG_STATE_LINK_OK;
3575                 }
3576                 break;
3577
3578         case ANEG_STATE_LINK_OK:
3579                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
3580                 ret = ANEG_DONE;
3581                 break;
3582
3583         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
3584                 /* ??? unimplemented */
3585                 break;
3586
3587         case ANEG_STATE_NEXT_PAGE_WAIT:
3588                 /* ??? unimplemented */
3589                 break;
3590
3591         default:
3592                 ret = ANEG_FAILED;
3593                 break;
3594         }
3595
3596         return ret;
3597 }
3598
3599 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
3600 {
3601         int res = 0;
3602         struct tg3_fiber_aneginfo aninfo;
3603         int status = ANEG_FAILED;
3604         unsigned int tick;
3605         u32 tmp;
3606
3607         tw32_f(MAC_TX_AUTO_NEG, 0);
3608
3609         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
3610         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
3611         udelay(40);
3612
3613         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
3614         udelay(40);
3615
3616         memset(&aninfo, 0, sizeof(aninfo));
3617         aninfo.flags |= MR_AN_ENABLE;
3618         aninfo.state = ANEG_STATE_UNKNOWN;
3619         aninfo.cur_time = 0;
3620         tick = 0;
3621         while (++tick < 195000) {
3622                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
3623                 if (status == ANEG_DONE || status == ANEG_FAILED)
3624                         break;
3625
3626                 udelay(1);
3627         }
3628
3629         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3630         tw32_f(MAC_MODE, tp->mac_mode);
3631         udelay(40);
3632
3633         *txflags = aninfo.txconfig;
3634         *rxflags = aninfo.flags;
3635
3636         if (status == ANEG_DONE &&
3637             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
3638                              MR_LP_ADV_FULL_DUPLEX)))
3639                 res = 1;
3640
3641         return res;
3642 }
3643
3644 static void tg3_init_bcm8002(struct tg3 *tp)
3645 {
3646         u32 mac_status = tr32(MAC_STATUS);
3647         int i;
3648
3649         /* Reset when initting first time or we have a link. */
3650         if ((tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) &&
3651             !(mac_status & MAC_STATUS_PCS_SYNCED))
3652                 return;
3653
3654         /* Set PLL lock range. */
3655         tg3_writephy(tp, 0x16, 0x8007);
3656
3657         /* SW reset */
3658         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
3659
3660         /* Wait for reset to complete. */
3661         /* XXX schedule_timeout() ... */
3662         for (i = 0; i < 500; i++)
3663                 udelay(10);
3664
3665         /* Config mode; select PMA/Ch 1 regs. */
3666         tg3_writephy(tp, 0x10, 0x8411);
3667
3668         /* Enable auto-lock and comdet, select txclk for tx. */
3669         tg3_writephy(tp, 0x11, 0x0a10);
3670
3671         tg3_writephy(tp, 0x18, 0x00a0);
3672         tg3_writephy(tp, 0x16, 0x41ff);
3673
3674         /* Assert and deassert POR. */
3675         tg3_writephy(tp, 0x13, 0x0400);
3676         udelay(40);
3677         tg3_writephy(tp, 0x13, 0x0000);
3678
3679         tg3_writephy(tp, 0x11, 0x0a50);
3680         udelay(40);
3681         tg3_writephy(tp, 0x11, 0x0a10);
3682
3683         /* Wait for signal to stabilize */
3684         /* XXX schedule_timeout() ... */
3685         for (i = 0; i < 15000; i++)
3686                 udelay(10);
3687
3688         /* Deselect the channel register so we can read the PHYID
3689          * later.
3690          */
3691         tg3_writephy(tp, 0x10, 0x8011);
3692 }
3693
3694 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
3695 {
3696         u16 flowctrl;
3697         u32 sg_dig_ctrl, sg_dig_status;
3698         u32 serdes_cfg, expected_sg_dig_ctrl;
3699         int workaround, port_a;
3700         int current_link_up;
3701
3702         serdes_cfg = 0;
3703         expected_sg_dig_ctrl = 0;
3704         workaround = 0;
3705         port_a = 1;
3706         current_link_up = 0;
3707
3708         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
3709             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
3710                 workaround = 1;
3711                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
3712                         port_a = 0;
3713
3714                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
3715                 /* preserve bits 20-23 for voltage regulator */
3716                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
3717         }
3718
3719         sg_dig_ctrl = tr32(SG_DIG_CTRL);
3720
3721         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
3722                 if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
3723                         if (workaround) {
3724                                 u32 val = serdes_cfg;
3725
3726                                 if (port_a)
3727                                         val |= 0xc010000;
3728                                 else
3729                                         val |= 0x4010000;
3730                                 tw32_f(MAC_SERDES_CFG, val);
3731                         }
3732
3733                         tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3734                 }
3735                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
3736                         tg3_setup_flow_control(tp, 0, 0);
3737                         current_link_up = 1;
3738                 }
3739                 goto out;
3740         }
3741
3742         /* Want auto-negotiation.  */
3743         expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
3744
3745         flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3746         if (flowctrl & ADVERTISE_1000XPAUSE)
3747                 expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
3748         if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3749                 expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
3750
3751         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
3752                 if ((tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT) &&
3753                     tp->serdes_counter &&
3754                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
3755                                     MAC_STATUS_RCVD_CFG)) ==
3756                      MAC_STATUS_PCS_SYNCED)) {
3757                         tp->serdes_counter--;
3758                         current_link_up = 1;
3759                         goto out;
3760                 }
3761 restart_autoneg:
3762                 if (workaround)
3763                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
3764                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
3765                 udelay(5);
3766                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
3767
3768                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
3769                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3770         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
3771                                  MAC_STATUS_SIGNAL_DET)) {
3772                 sg_dig_status = tr32(SG_DIG_STATUS);
3773                 mac_status = tr32(MAC_STATUS);
3774
3775                 if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
3776                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
3777                         u32 local_adv = 0, remote_adv = 0;
3778
3779                         if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
3780                                 local_adv |= ADVERTISE_1000XPAUSE;
3781                         if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
3782                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
3783
3784                         if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
3785                                 remote_adv |= LPA_1000XPAUSE;
3786                         if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
3787                                 remote_adv |= LPA_1000XPAUSE_ASYM;
3788
3789                         tg3_setup_flow_control(tp, local_adv, remote_adv);
3790                         current_link_up = 1;
3791                         tp->serdes_counter = 0;
3792                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3793                 } else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
3794                         if (tp->serdes_counter)
3795                                 tp->serdes_counter--;
3796                         else {
3797                                 if (workaround) {
3798                                         u32 val = serdes_cfg;
3799
3800                                         if (port_a)
3801                                                 val |= 0xc010000;
3802                                         else
3803                                                 val |= 0x4010000;
3804
3805                                         tw32_f(MAC_SERDES_CFG, val);
3806                                 }
3807
3808                                 tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3809                                 udelay(40);
3810
3811                                 /* Link parallel detection - link is up */
3812                                 /* only if we have PCS_SYNC and not */
3813                                 /* receiving config code words */
3814                                 mac_status = tr32(MAC_STATUS);
3815                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
3816                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
3817                                         tg3_setup_flow_control(tp, 0, 0);
3818                                         current_link_up = 1;
3819                                         tp->tg3_flags2 |=
3820                                                 TG3_FLG2_PARALLEL_DETECT;
3821                                         tp->serdes_counter =
3822                                                 SERDES_PARALLEL_DET_TIMEOUT;
3823                                 } else
3824                                         goto restart_autoneg;
3825                         }
3826                 }
3827         } else {
3828                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
3829                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
3830         }
3831
3832 out:
3833         return current_link_up;
3834 }
3835
3836 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
3837 {
3838         int current_link_up = 0;
3839
3840         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
3841                 goto out;
3842
3843         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3844                 u32 txflags, rxflags;
3845                 int i;
3846
3847                 if (fiber_autoneg(tp, &txflags, &rxflags)) {
3848                         u32 local_adv = 0, remote_adv = 0;
3849
3850                         if (txflags & ANEG_CFG_PS1)
3851                                 local_adv |= ADVERTISE_1000XPAUSE;
3852                         if (txflags & ANEG_CFG_PS2)
3853                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
3854
3855                         if (rxflags & MR_LP_ADV_SYM_PAUSE)
3856                                 remote_adv |= LPA_1000XPAUSE;
3857                         if (rxflags & MR_LP_ADV_ASYM_PAUSE)
3858                                 remote_adv |= LPA_1000XPAUSE_ASYM;
3859
3860                         tg3_setup_flow_control(tp, local_adv, remote_adv);
3861
3862                         current_link_up = 1;
3863                 }
3864                 for (i = 0; i < 30; i++) {
3865                         udelay(20);
3866                         tw32_f(MAC_STATUS,
3867                                (MAC_STATUS_SYNC_CHANGED |
3868                                 MAC_STATUS_CFG_CHANGED));
3869                         udelay(40);
3870                         if ((tr32(MAC_STATUS) &
3871                              (MAC_STATUS_SYNC_CHANGED |
3872                               MAC_STATUS_CFG_CHANGED)) == 0)
3873                                 break;
3874                 }
3875
3876                 mac_status = tr32(MAC_STATUS);
3877                 if (current_link_up == 0 &&
3878                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
3879                     !(mac_status & MAC_STATUS_RCVD_CFG))
3880                         current_link_up = 1;
3881         } else {
3882                 tg3_setup_flow_control(tp, 0, 0);
3883
3884                 /* Forcing 1000FD link up. */
3885                 current_link_up = 1;
3886
3887                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
3888                 udelay(40);
3889
3890                 tw32_f(MAC_MODE, tp->mac_mode);
3891                 udelay(40);
3892         }
3893
3894 out:
3895         return current_link_up;
3896 }
3897
3898 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
3899 {
3900         u32 orig_pause_cfg;
3901         u16 orig_active_speed;
3902         u8 orig_active_duplex;
3903         u32 mac_status;
3904         int current_link_up;
3905         int i;
3906
3907         orig_pause_cfg = tp->link_config.active_flowctrl;
3908         orig_active_speed = tp->link_config.active_speed;
3909         orig_active_duplex = tp->link_config.active_duplex;
3910
3911         if (!(tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG) &&
3912             netif_carrier_ok(tp->dev) &&
3913             (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE)) {
3914                 mac_status = tr32(MAC_STATUS);
3915                 mac_status &= (MAC_STATUS_PCS_SYNCED |
3916                                MAC_STATUS_SIGNAL_DET |
3917                                MAC_STATUS_CFG_CHANGED |
3918                                MAC_STATUS_RCVD_CFG);
3919                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
3920                                    MAC_STATUS_SIGNAL_DET)) {
3921                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
3922                                             MAC_STATUS_CFG_CHANGED));
3923                         return 0;
3924                 }
3925         }
3926
3927         tw32_f(MAC_TX_AUTO_NEG, 0);
3928
3929         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
3930         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
3931         tw32_f(MAC_MODE, tp->mac_mode);
3932         udelay(40);
3933
3934         if (tp->phy_id == PHY_ID_BCM8002)
3935                 tg3_init_bcm8002(tp);
3936
3937         /* Enable link change event even when serdes polling.  */
3938         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3939         udelay(40);
3940
3941         current_link_up = 0;
3942         mac_status = tr32(MAC_STATUS);
3943
3944         if (tp->tg3_flags2 & TG3_FLG2_HW_AUTONEG)
3945                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
3946         else
3947                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
3948
3949         tp->napi[0].hw_status->status =
3950                 (SD_STATUS_UPDATED |
3951                  (tp->napi[0].hw_status->status & ~SD_STATUS_LINK_CHG));
3952
3953         for (i = 0; i < 100; i++) {
3954                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
3955                                     MAC_STATUS_CFG_CHANGED));
3956                 udelay(5);
3957                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
3958                                          MAC_STATUS_CFG_CHANGED |
3959                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
3960                         break;
3961         }
3962
3963         mac_status = tr32(MAC_STATUS);
3964         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
3965                 current_link_up = 0;
3966                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
3967                     tp->serdes_counter == 0) {
3968                         tw32_f(MAC_MODE, (tp->mac_mode |
3969                                           MAC_MODE_SEND_CONFIGS));
3970                         udelay(1);
3971                         tw32_f(MAC_MODE, tp->mac_mode);
3972                 }
3973         }
3974
3975         if (current_link_up == 1) {
3976                 tp->link_config.active_speed = SPEED_1000;
3977                 tp->link_config.active_duplex = DUPLEX_FULL;
3978                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
3979                                     LED_CTRL_LNKLED_OVERRIDE |
3980                                     LED_CTRL_1000MBPS_ON));
3981         } else {
3982                 tp->link_config.active_speed = SPEED_INVALID;
3983                 tp->link_config.active_duplex = DUPLEX_INVALID;
3984                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
3985                                     LED_CTRL_LNKLED_OVERRIDE |
3986                                     LED_CTRL_TRAFFIC_OVERRIDE));
3987         }
3988
3989         if (current_link_up != netif_carrier_ok(tp->dev)) {
3990                 if (current_link_up)
3991                         netif_carrier_on(tp->dev);
3992                 else
3993                         netif_carrier_off(tp->dev);
3994                 tg3_link_report(tp);
3995         } else {
3996                 u32 now_pause_cfg = tp->link_config.active_flowctrl;
3997                 if (orig_pause_cfg != now_pause_cfg ||
3998                     orig_active_speed != tp->link_config.active_speed ||
3999                     orig_active_duplex != tp->link_config.active_duplex)
4000                         tg3_link_report(tp);
4001         }
4002
4003         return 0;
4004 }
4005
4006 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
4007 {
4008         int current_link_up, err = 0;
4009         u32 bmsr, bmcr;
4010         u16 current_speed;
4011         u8 current_duplex;
4012         u32 local_adv, remote_adv;
4013
4014         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4015         tw32_f(MAC_MODE, tp->mac_mode);
4016         udelay(40);
4017
4018         tw32(MAC_EVENT, 0);
4019
4020         tw32_f(MAC_STATUS,
4021              (MAC_STATUS_SYNC_CHANGED |
4022               MAC_STATUS_CFG_CHANGED |
4023               MAC_STATUS_MI_COMPLETION |
4024               MAC_STATUS_LNKSTATE_CHANGED));
4025         udelay(40);
4026
4027         if (force_reset)
4028                 tg3_phy_reset(tp);
4029
4030         current_link_up = 0;
4031         current_speed = SPEED_INVALID;
4032         current_duplex = DUPLEX_INVALID;
4033
4034         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4035         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4036         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
4037                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4038                         bmsr |= BMSR_LSTATUS;
4039                 else
4040                         bmsr &= ~BMSR_LSTATUS;
4041         }
4042
4043         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
4044
4045         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
4046             (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
4047                 /* do nothing, just check for link up at the end */
4048         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4049                 u32 adv, new_adv;
4050
4051                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4052                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
4053                                   ADVERTISE_1000XPAUSE |
4054                                   ADVERTISE_1000XPSE_ASYM |
4055                                   ADVERTISE_SLCT);
4056
4057                 new_adv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
4058
4059                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
4060                         new_adv |= ADVERTISE_1000XHALF;
4061                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
4062                         new_adv |= ADVERTISE_1000XFULL;
4063
4064                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
4065                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
4066                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
4067                         tg3_writephy(tp, MII_BMCR, bmcr);
4068
4069                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4070                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
4071                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4072
4073                         return err;
4074                 }
4075         } else {
4076                 u32 new_bmcr;
4077
4078                 bmcr &= ~BMCR_SPEED1000;
4079                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
4080
4081                 if (tp->link_config.duplex == DUPLEX_FULL)
4082                         new_bmcr |= BMCR_FULLDPLX;
4083
4084                 if (new_bmcr != bmcr) {
4085                         /* BMCR_SPEED1000 is a reserved bit that needs
4086                          * to be set on write.
4087                          */
4088                         new_bmcr |= BMCR_SPEED1000;
4089
4090                         /* Force a linkdown */
4091                         if (netif_carrier_ok(tp->dev)) {
4092                                 u32 adv;
4093
4094                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4095                                 adv &= ~(ADVERTISE_1000XFULL |
4096                                          ADVERTISE_1000XHALF |
4097                                          ADVERTISE_SLCT);
4098                                 tg3_writephy(tp, MII_ADVERTISE, adv);
4099                                 tg3_writephy(tp, MII_BMCR, bmcr |
4100                                                            BMCR_ANRESTART |
4101                                                            BMCR_ANENABLE);
4102                                 udelay(10);
4103                                 netif_carrier_off(tp->dev);
4104                         }
4105                         tg3_writephy(tp, MII_BMCR, new_bmcr);
4106                         bmcr = new_bmcr;
4107                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4108                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4109                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
4110                             ASIC_REV_5714) {
4111                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4112                                         bmsr |= BMSR_LSTATUS;
4113                                 else
4114                                         bmsr &= ~BMSR_LSTATUS;
4115                         }
4116                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4117                 }
4118         }
4119
4120         if (bmsr & BMSR_LSTATUS) {
4121                 current_speed = SPEED_1000;
4122                 current_link_up = 1;
4123                 if (bmcr & BMCR_FULLDPLX)
4124                         current_duplex = DUPLEX_FULL;
4125                 else
4126                         current_duplex = DUPLEX_HALF;
4127
4128                 local_adv = 0;
4129                 remote_adv = 0;
4130
4131                 if (bmcr & BMCR_ANENABLE) {
4132                         u32 common;
4133
4134                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
4135                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
4136                         common = local_adv & remote_adv;
4137                         if (common & (ADVERTISE_1000XHALF |
4138                                       ADVERTISE_1000XFULL)) {
4139                                 if (common & ADVERTISE_1000XFULL)
4140                                         current_duplex = DUPLEX_FULL;
4141                                 else
4142                                         current_duplex = DUPLEX_HALF;
4143                         }
4144                         else
4145                                 current_link_up = 0;
4146                 }
4147         }
4148
4149         if (current_link_up == 1 && current_duplex == DUPLEX_FULL)
4150                 tg3_setup_flow_control(tp, local_adv, remote_adv);
4151
4152         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
4153         if (tp->link_config.active_duplex == DUPLEX_HALF)
4154                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
4155
4156         tw32_f(MAC_MODE, tp->mac_mode);
4157         udelay(40);
4158
4159         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4160
4161         tp->link_config.active_speed = current_speed;
4162         tp->link_config.active_duplex = current_duplex;
4163
4164         if (current_link_up != netif_carrier_ok(tp->dev)) {
4165                 if (current_link_up)
4166                         netif_carrier_on(tp->dev);
4167                 else {
4168                         netif_carrier_off(tp->dev);
4169                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4170                 }
4171                 tg3_link_report(tp);
4172         }
4173         return err;
4174 }
4175
4176 static void tg3_serdes_parallel_detect(struct tg3 *tp)
4177 {
4178         if (tp->serdes_counter) {
4179                 /* Give autoneg time to complete. */
4180                 tp->serdes_counter--;
4181                 return;
4182         }
4183         if (!netif_carrier_ok(tp->dev) &&
4184             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
4185                 u32 bmcr;
4186
4187                 tg3_readphy(tp, MII_BMCR, &bmcr);
4188                 if (bmcr & BMCR_ANENABLE) {
4189                         u32 phy1, phy2;
4190
4191                         /* Select shadow register 0x1f */
4192                         tg3_writephy(tp, 0x1c, 0x7c00);
4193                         tg3_readphy(tp, 0x1c, &phy1);
4194
4195                         /* Select expansion interrupt status register */
4196                         tg3_writephy(tp, 0x17, 0x0f01);
4197                         tg3_readphy(tp, 0x15, &phy2);
4198                         tg3_readphy(tp, 0x15, &phy2);
4199
4200                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
4201                                 /* We have signal detect and not receiving
4202                                  * config code words, link is up by parallel
4203                                  * detection.
4204                                  */
4205
4206                                 bmcr &= ~BMCR_ANENABLE;
4207                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
4208                                 tg3_writephy(tp, MII_BMCR, bmcr);
4209                                 tp->tg3_flags2 |= TG3_FLG2_PARALLEL_DETECT;
4210                         }
4211                 }
4212         }
4213         else if (netif_carrier_ok(tp->dev) &&
4214                  (tp->link_config.autoneg == AUTONEG_ENABLE) &&
4215                  (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT)) {
4216                 u32 phy2;
4217
4218                 /* Select expansion interrupt status register */
4219                 tg3_writephy(tp, 0x17, 0x0f01);
4220                 tg3_readphy(tp, 0x15, &phy2);
4221                 if (phy2 & 0x20) {
4222                         u32 bmcr;
4223
4224                         /* Config code words received, turn on autoneg. */
4225                         tg3_readphy(tp, MII_BMCR, &bmcr);
4226                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
4227
4228                         tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
4229
4230                 }
4231         }
4232 }
4233
4234 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
4235 {
4236         int err;
4237
4238         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
4239                 err = tg3_setup_fiber_phy(tp, force_reset);
4240         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
4241                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
4242         } else {
4243                 err = tg3_setup_copper_phy(tp, force_reset);
4244         }
4245
4246         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX) {
4247                 u32 val, scale;
4248
4249                 val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
4250                 if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
4251                         scale = 65;
4252                 else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
4253                         scale = 6;
4254                 else
4255                         scale = 12;
4256
4257                 val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
4258                 val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
4259                 tw32(GRC_MISC_CFG, val);
4260         }
4261
4262         if (tp->link_config.active_speed == SPEED_1000 &&
4263             tp->link_config.active_duplex == DUPLEX_HALF)
4264                 tw32(MAC_TX_LENGTHS,
4265                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4266                       (6 << TX_LENGTHS_IPG_SHIFT) |
4267                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
4268         else
4269                 tw32(MAC_TX_LENGTHS,
4270                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4271                       (6 << TX_LENGTHS_IPG_SHIFT) |
4272                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
4273
4274         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
4275                 if (netif_carrier_ok(tp->dev)) {
4276                         tw32(HOSTCC_STAT_COAL_TICKS,
4277                              tp->coal.stats_block_coalesce_usecs);
4278                 } else {
4279                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
4280                 }
4281         }
4282
4283         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND) {
4284                 u32 val = tr32(PCIE_PWR_MGMT_THRESH);
4285                 if (!netif_carrier_ok(tp->dev))
4286                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
4287                               tp->pwrmgmt_thresh;
4288                 else
4289                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
4290                 tw32(PCIE_PWR_MGMT_THRESH, val);
4291         }
4292
4293         return err;
4294 }
4295
4296 /* This is called whenever we suspect that the system chipset is re-
4297  * ordering the sequence of MMIO to the tx send mailbox. The symptom
4298  * is bogus tx completions. We try to recover by setting the
4299  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
4300  * in the workqueue.
4301  */
4302 static void tg3_tx_recover(struct tg3 *tp)
4303 {
4304         BUG_ON((tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER) ||
4305                tp->write32_tx_mbox == tg3_write_indirect_mbox);
4306
4307         printk(KERN_WARNING PFX "%s: The system may be re-ordering memory-"
4308                "mapped I/O cycles to the network device, attempting to "
4309                "recover. Please report the problem to the driver maintainer "
4310                "and include system chipset information.\n", tp->dev->name);
4311
4312         spin_lock(&tp->lock);
4313         tp->tg3_flags |= TG3_FLAG_TX_RECOVERY_PENDING;
4314         spin_unlock(&tp->lock);
4315 }
4316
4317 static inline u32 tg3_tx_avail(struct tg3_napi *tnapi)
4318 {
4319         smp_mb();
4320         return tnapi->tx_pending -
4321                ((tnapi->tx_prod - tnapi->tx_cons) & (TG3_TX_RING_SIZE - 1));
4322 }
4323
4324 /* Tigon3 never reports partial packet sends.  So we do not
4325  * need special logic to handle SKBs that have not had all
4326  * of their frags sent yet, like SunGEM does.
4327  */
4328 static void tg3_tx(struct tg3_napi *tnapi)
4329 {
4330         struct tg3 *tp = tnapi->tp;
4331         u32 hw_idx = tnapi->hw_status->idx[0].tx_consumer;
4332         u32 sw_idx = tnapi->tx_cons;
4333         struct netdev_queue *txq;
4334         int index = tnapi - tp->napi;
4335
4336         if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX)
4337                 index--;
4338
4339         txq = netdev_get_tx_queue(tp->dev, index);
4340
4341         while (sw_idx != hw_idx) {
4342                 struct tx_ring_info *ri = &tnapi->tx_buffers[sw_idx];
4343                 struct sk_buff *skb = ri->skb;
4344                 int i, tx_bug = 0;
4345
4346                 if (unlikely(skb == NULL)) {
4347                         tg3_tx_recover(tp);
4348                         return;
4349                 }
4350
4351                 skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE);
4352
4353                 ri->skb = NULL;
4354
4355                 sw_idx = NEXT_TX(sw_idx);
4356
4357                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4358                         ri = &tnapi->tx_buffers[sw_idx];
4359                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
4360                                 tx_bug = 1;
4361                         sw_idx = NEXT_TX(sw_idx);
4362                 }
4363
4364                 dev_kfree_skb(skb);
4365
4366                 if (unlikely(tx_bug)) {
4367                         tg3_tx_recover(tp);
4368                         return;
4369                 }
4370         }
4371
4372         tnapi->tx_cons = sw_idx;
4373
4374         /* Need to make the tx_cons update visible to tg3_start_xmit()
4375          * before checking for netif_queue_stopped().  Without the
4376          * memory barrier, there is a small possibility that tg3_start_xmit()
4377          * will miss it and cause the queue to be stopped forever.
4378          */
4379         smp_mb();
4380
4381         if (unlikely(netif_tx_queue_stopped(txq) &&
4382                      (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))) {
4383                 __netif_tx_lock(txq, smp_processor_id());
4384                 if (netif_tx_queue_stopped(txq) &&
4385                     (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))
4386                         netif_tx_wake_queue(txq);
4387                 __netif_tx_unlock(txq);
4388         }
4389 }
4390
4391 /* Returns size of skb allocated or < 0 on error.
4392  *
4393  * We only need to fill in the address because the other members
4394  * of the RX descriptor are invariant, see tg3_init_rings.
4395  *
4396  * Note the purposeful assymetry of cpu vs. chip accesses.  For
4397  * posting buffers we only dirty the first cache line of the RX
4398  * descriptor (containing the address).  Whereas for the RX status
4399  * buffers the cpu only reads the last cacheline of the RX descriptor
4400  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
4401  */
4402 static int tg3_alloc_rx_skb(struct tg3_napi *tnapi, u32 opaque_key,
4403                             int src_idx, u32 dest_idx_unmasked)
4404 {
4405         struct tg3 *tp = tnapi->tp;
4406         struct tg3_rx_buffer_desc *desc;
4407         struct ring_info *map, *src_map;
4408         struct sk_buff *skb;
4409         dma_addr_t mapping;
4410         int skb_size, dest_idx;
4411         struct tg3_rx_prodring_set *tpr = &tp->prodring[0];
4412
4413         src_map = NULL;
4414         switch (opaque_key) {
4415         case RXD_OPAQUE_RING_STD:
4416                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
4417                 desc = &tpr->rx_std[dest_idx];
4418                 map = &tpr->rx_std_buffers[dest_idx];
4419                 if (src_idx >= 0)
4420                         src_map = &tpr->rx_std_buffers[src_idx];
4421                 skb_size = tp->rx_pkt_map_sz;
4422                 break;
4423
4424         case RXD_OPAQUE_RING_JUMBO:
4425                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
4426                 desc = &tpr->rx_jmb[dest_idx].std;
4427                 map = &tpr->rx_jmb_buffers[dest_idx];
4428                 if (src_idx >= 0)
4429                         src_map = &tpr->rx_jmb_buffers[src_idx];
4430                 skb_size = TG3_RX_JMB_MAP_SZ;
4431                 break;
4432
4433         default:
4434                 return -EINVAL;
4435         }
4436
4437         /* Do not overwrite any of the map or rp information
4438          * until we are sure we can commit to a new buffer.
4439          *
4440          * Callers depend upon this behavior and assume that
4441          * we leave everything unchanged if we fail.
4442          */
4443         skb = netdev_alloc_skb(tp->dev, skb_size + tp->rx_offset);
4444         if (skb == NULL)
4445                 return -ENOMEM;
4446
4447         skb_reserve(skb, tp->rx_offset);
4448
4449         mapping = pci_map_single(tp->pdev, skb->data, skb_size,
4450                                  PCI_DMA_FROMDEVICE);
4451
4452         map->skb = skb;
4453         pci_unmap_addr_set(map, mapping, mapping);
4454
4455         if (src_map != NULL)
4456                 src_map->skb = NULL;
4457
4458         desc->addr_hi = ((u64)mapping >> 32);
4459         desc->addr_lo = ((u64)mapping & 0xffffffff);
4460
4461         return skb_size;
4462 }
4463
4464 /* We only need to move over in the address because the other
4465  * members of the RX descriptor are invariant.  See notes above
4466  * tg3_alloc_rx_skb for full details.
4467  */
4468 static void tg3_recycle_rx(struct tg3_napi *tnapi, u32 opaque_key,
4469                            int src_idx, u32 dest_idx_unmasked)
4470 {
4471         struct tg3 *tp = tnapi->tp;
4472         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
4473         struct ring_info *src_map, *dest_map;
4474         int dest_idx;
4475         struct tg3_rx_prodring_set *tpr = &tp->prodring[0];
4476
4477         switch (opaque_key) {
4478         case RXD_OPAQUE_RING_STD:
4479                 dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
4480                 dest_desc = &tpr->rx_std[dest_idx];
4481                 dest_map = &tpr->rx_std_buffers[dest_idx];
4482                 src_desc = &tpr->rx_std[src_idx];
4483                 src_map = &tpr->rx_std_buffers[src_idx];
4484                 break;
4485
4486         case RXD_OPAQUE_RING_JUMBO:
4487                 dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
4488                 dest_desc = &tpr->rx_jmb[dest_idx].std;
4489                 dest_map = &tpr->rx_jmb_buffers[dest_idx];
4490                 src_desc = &tpr->rx_jmb[src_idx].std;
4491                 src_map = &tpr->rx_jmb_buffers[src_idx];
4492                 break;
4493
4494         default:
4495                 return;
4496         }
4497
4498         dest_map->skb = src_map->skb;
4499         pci_unmap_addr_set(dest_map, mapping,
4500                            pci_unmap_addr(src_map, mapping));
4501         dest_desc->addr_hi = src_desc->addr_hi;
4502         dest_desc->addr_lo = src_desc->addr_lo;
4503
4504         src_map->skb = NULL;
4505 }
4506
4507 /* The RX ring scheme is composed of multiple rings which post fresh
4508  * buffers to the chip, and one special ring the chip uses to report
4509  * status back to the host.
4510  *
4511  * The special ring reports the status of received packets to the
4512  * host.  The chip does not write into the original descriptor the
4513  * RX buffer was obtained from.  The chip simply takes the original
4514  * descriptor as provided by the host, updates the status and length
4515  * field, then writes this into the next status ring entry.
4516  *
4517  * Each ring the host uses to post buffers to the chip is described
4518  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
4519  * it is first placed into the on-chip ram.  When the packet's length
4520  * is known, it walks down the TG3_BDINFO entries to select the ring.
4521  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
4522  * which is within the range of the new packet's length is chosen.
4523  *
4524  * The "separate ring for rx status" scheme may sound queer, but it makes
4525  * sense from a cache coherency perspective.  If only the host writes
4526  * to the buffer post rings, and only the chip writes to the rx status
4527  * rings, then cache lines never move beyond shared-modified state.
4528  * If both the host and chip were to write into the same ring, cache line
4529  * eviction could occur since both entities want it in an exclusive state.
4530  */
4531 static int tg3_rx(struct tg3_napi *tnapi, int budget)
4532 {
4533         struct tg3 *tp = tnapi->tp;
4534         u32 work_mask, rx_std_posted = 0;
4535         u32 sw_idx = tnapi->rx_rcb_ptr;
4536         u16 hw_idx;
4537         int received;
4538         struct tg3_rx_prodring_set *tpr = &tp->prodring[0];
4539
4540         hw_idx = *(tnapi->rx_rcb_prod_idx);
4541         /*
4542          * We need to order the read of hw_idx and the read of
4543          * the opaque cookie.
4544          */
4545         rmb();
4546         work_mask = 0;
4547         received = 0;
4548         while (sw_idx != hw_idx && budget > 0) {
4549                 struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx];
4550                 unsigned int len;
4551                 struct sk_buff *skb;
4552                 dma_addr_t dma_addr;
4553                 u32 opaque_key, desc_idx, *post_ptr;
4554
4555                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
4556                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
4557                 if (opaque_key == RXD_OPAQUE_RING_STD) {
4558                         struct ring_info *ri = &tpr->rx_std_buffers[desc_idx];
4559                         dma_addr = pci_unmap_addr(ri, mapping);
4560                         skb = ri->skb;
4561                         post_ptr = &tpr->rx_std_ptr;
4562                         rx_std_posted++;
4563                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
4564                         struct ring_info *ri = &tpr->rx_jmb_buffers[desc_idx];
4565                         dma_addr = pci_unmap_addr(ri, mapping);
4566                         skb = ri->skb;
4567                         post_ptr = &tpr->rx_jmb_ptr;
4568                 } else
4569                         goto next_pkt_nopost;
4570
4571                 work_mask |= opaque_key;
4572
4573                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
4574                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
4575                 drop_it:
4576                         tg3_recycle_rx(tnapi, opaque_key,
4577                                        desc_idx, *post_ptr);
4578                 drop_it_no_recycle:
4579                         /* Other statistics kept track of by card. */
4580                         tp->net_stats.rx_dropped++;
4581                         goto next_pkt;
4582                 }
4583
4584                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
4585                       ETH_FCS_LEN;
4586
4587                 if (len > RX_COPY_THRESHOLD
4588                         && tp->rx_offset == NET_IP_ALIGN
4589                         /* rx_offset will likely not equal NET_IP_ALIGN
4590                          * if this is a 5701 card running in PCI-X mode
4591                          * [see tg3_get_invariants()]
4592                          */
4593                 ) {
4594                         int skb_size;
4595
4596                         skb_size = tg3_alloc_rx_skb(tnapi, opaque_key,
4597                                                     desc_idx, *post_ptr);
4598                         if (skb_size < 0)
4599                                 goto drop_it;
4600
4601                         pci_unmap_single(tp->pdev, dma_addr, skb_size,
4602                                          PCI_DMA_FROMDEVICE);
4603
4604                         skb_put(skb, len);
4605                 } else {
4606                         struct sk_buff *copy_skb;
4607
4608                         tg3_recycle_rx(tnapi, opaque_key,
4609                                        desc_idx, *post_ptr);
4610
4611                         copy_skb = netdev_alloc_skb(tp->dev,
4612                                                     len + TG3_RAW_IP_ALIGN);
4613                         if (copy_skb == NULL)
4614                                 goto drop_it_no_recycle;
4615
4616                         skb_reserve(copy_skb, TG3_RAW_IP_ALIGN);
4617                         skb_put(copy_skb, len);
4618                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4619                         skb_copy_from_linear_data(skb, copy_skb->data, len);
4620                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4621
4622                         /* We'll reuse the original ring buffer. */
4623                         skb = copy_skb;
4624                 }
4625
4626                 if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
4627                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
4628                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
4629                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
4630                         skb->ip_summed = CHECKSUM_UNNECESSARY;
4631                 else
4632                         skb->ip_summed = CHECKSUM_NONE;
4633
4634                 skb->protocol = eth_type_trans(skb, tp->dev);
4635
4636                 if (len > (tp->dev->mtu + ETH_HLEN) &&
4637                     skb->protocol != htons(ETH_P_8021Q)) {
4638                         dev_kfree_skb(skb);
4639                         goto next_pkt;
4640                 }
4641
4642 #if TG3_VLAN_TAG_USED
4643                 if (tp->vlgrp != NULL &&
4644                     desc->type_flags & RXD_FLAG_VLAN) {
4645                         vlan_gro_receive(&tnapi->napi, tp->vlgrp,
4646                                          desc->err_vlan & RXD_VLAN_MASK, skb);
4647                 } else
4648 #endif
4649                         napi_gro_receive(&tnapi->napi, skb);
4650
4651                 received++;
4652                 budget--;
4653
4654 next_pkt:
4655                 (*post_ptr)++;
4656
4657                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
4658                         u32 idx = *post_ptr % TG3_RX_RING_SIZE;
4659
4660                         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX +
4661                                      TG3_64BIT_REG_LOW, idx);
4662                         work_mask &= ~RXD_OPAQUE_RING_STD;
4663                         rx_std_posted = 0;
4664                 }
4665 next_pkt_nopost:
4666                 sw_idx++;
4667                 sw_idx &= (TG3_RX_RCB_RING_SIZE(tp) - 1);
4668
4669                 /* Refresh hw_idx to see if there is new work */
4670                 if (sw_idx == hw_idx) {
4671                         hw_idx = *(tnapi->rx_rcb_prod_idx);
4672                         rmb();
4673                 }
4674         }
4675
4676         /* ACK the status ring. */
4677         tnapi->rx_rcb_ptr = sw_idx;
4678         tw32_rx_mbox(tnapi->consmbox, sw_idx);
4679
4680         /* Refill RX ring(s). */
4681         if (work_mask & RXD_OPAQUE_RING_STD) {
4682                 sw_idx = tpr->rx_std_ptr % TG3_RX_RING_SIZE;
4683                 tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
4684                              sw_idx);
4685         }
4686         if (work_mask & RXD_OPAQUE_RING_JUMBO) {
4687                 sw_idx = tpr->rx_jmb_ptr % TG3_RX_JUMBO_RING_SIZE;
4688                 tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
4689                              sw_idx);
4690         }
4691         mmiowb();
4692
4693         return received;
4694 }
4695
4696 static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget)
4697 {
4698         struct tg3 *tp = tnapi->tp;
4699         struct tg3_hw_status *sblk = tnapi->hw_status;
4700
4701         /* handle link change and other phy events */
4702         if (!(tp->tg3_flags &
4703               (TG3_FLAG_USE_LINKCHG_REG |
4704                TG3_FLAG_POLL_SERDES))) {
4705                 if (sblk->status & SD_STATUS_LINK_CHG) {
4706                         sblk->status = SD_STATUS_UPDATED |
4707                                 (sblk->status & ~SD_STATUS_LINK_CHG);
4708                         spin_lock(&tp->lock);
4709                         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
4710                                 tw32_f(MAC_STATUS,
4711                                      (MAC_STATUS_SYNC_CHANGED |
4712                                       MAC_STATUS_CFG_CHANGED |
4713                                       MAC_STATUS_MI_COMPLETION |
4714                                       MAC_STATUS_LNKSTATE_CHANGED));
4715                                 udelay(40);
4716                         } else
4717                                 tg3_setup_phy(tp, 0);
4718                         spin_unlock(&tp->lock);
4719                 }
4720         }
4721
4722         /* run TX completion thread */
4723         if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) {
4724                 tg3_tx(tnapi);
4725                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
4726                         return work_done;
4727         }
4728
4729         /* run RX thread, within the bounds set by NAPI.
4730          * All RX "locking" is done by ensuring outside
4731          * code synchronizes with tg3->napi.poll()
4732          */
4733         if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
4734                 work_done += tg3_rx(tnapi, budget - work_done);
4735
4736         return work_done;
4737 }
4738
4739 static int tg3_poll(struct napi_struct *napi, int budget)
4740 {
4741         struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
4742         struct tg3 *tp = tnapi->tp;
4743         int work_done = 0;
4744         struct tg3_hw_status *sblk = tnapi->hw_status;
4745
4746         while (1) {
4747                 work_done = tg3_poll_work(tnapi, work_done, budget);
4748
4749                 if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING))
4750                         goto tx_recovery;
4751
4752                 if (unlikely(work_done >= budget))
4753                         break;
4754
4755                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
4756                         /* tp->last_tag is used in tg3_int_reenable() below
4757                          * to tell the hw how much work has been processed,
4758                          * so we must read it before checking for more work.
4759                          */
4760                         tnapi->last_tag = sblk->status_tag;
4761                         tnapi->last_irq_tag = tnapi->last_tag;
4762                         rmb();
4763                 } else
4764                         sblk->status &= ~SD_STATUS_UPDATED;
4765
4766                 if (likely(!tg3_has_work(tnapi))) {
4767                         napi_complete(napi);
4768                         tg3_int_reenable(tnapi);
4769                         break;
4770                 }
4771         }
4772
4773         return work_done;
4774
4775 tx_recovery:
4776         /* work_done is guaranteed to be less than budget. */
4777         napi_complete(napi);
4778         schedule_work(&tp->reset_task);
4779         return work_done;
4780 }
4781
4782 static void tg3_irq_quiesce(struct tg3 *tp)
4783 {
4784         int i;
4785
4786         BUG_ON(tp->irq_sync);
4787
4788         tp->irq_sync = 1;
4789         smp_mb();
4790
4791         for (i = 0; i < tp->irq_cnt; i++)
4792                 synchronize_irq(tp->napi[i].irq_vec);
4793 }
4794
4795 static inline int tg3_irq_sync(struct tg3 *tp)
4796 {
4797         return tp->irq_sync;
4798 }
4799
4800 /* Fully shutdown all tg3 driver activity elsewhere in the system.
4801  * If irq_sync is non-zero, then the IRQ handler must be synchronized
4802  * with as well.  Most of the time, this is not necessary except when
4803  * shutting down the device.
4804  */
4805 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
4806 {
4807         spin_lock_bh(&tp->lock);
4808         if (irq_sync)
4809                 tg3_irq_quiesce(tp);
4810 }
4811
4812 static inline void tg3_full_unlock(struct tg3 *tp)
4813 {
4814         spin_unlock_bh(&tp->lock);
4815 }
4816
4817 /* One-shot MSI handler - Chip automatically disables interrupt
4818  * after sending MSI so driver doesn't have to do it.
4819  */
4820 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
4821 {
4822         struct tg3_napi *tnapi = dev_id;
4823         struct tg3 *tp = tnapi->tp;
4824
4825         prefetch(tnapi->hw_status);
4826         if (tnapi->rx_rcb)
4827                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
4828
4829         if (likely(!tg3_irq_sync(tp)))
4830                 napi_schedule(&tnapi->napi);
4831
4832         return IRQ_HANDLED;
4833 }
4834
4835 /* MSI ISR - No need to check for interrupt sharing and no need to
4836  * flush status block and interrupt mailbox. PCI ordering rules
4837  * guarantee that MSI will arrive after the status block.
4838  */
4839 static irqreturn_t tg3_msi(int irq, void *dev_id)
4840 {
4841         struct tg3_napi *tnapi = dev_id;
4842         struct tg3 *tp = tnapi->tp;
4843
4844         prefetch(tnapi->hw_status);
4845         if (tnapi->rx_rcb)
4846                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
4847         /*
4848          * Writing any value to intr-mbox-0 clears PCI INTA# and
4849          * chip-internal interrupt pending events.
4850          * Writing non-zero to intr-mbox-0 additional tells the
4851          * NIC to stop sending us irqs, engaging "in-intr-handler"
4852          * event coalescing.
4853          */
4854         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
4855         if (likely(!tg3_irq_sync(tp)))
4856                 napi_schedule(&tnapi->napi);
4857
4858         return IRQ_RETVAL(1);
4859 }
4860
4861 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
4862 {
4863         struct tg3_napi *tnapi = dev_id;
4864         struct tg3 *tp = tnapi->tp;
4865         struct tg3_hw_status *sblk = tnapi->hw_status;
4866         unsigned int handled = 1;
4867
4868         /* In INTx mode, it is possible for the interrupt to arrive at
4869          * the CPU before the status block posted prior to the interrupt.
4870          * Reading the PCI State register will confirm whether the
4871          * interrupt is ours and will flush the status block.
4872          */
4873         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
4874                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
4875                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
4876                         handled = 0;
4877                         goto out;
4878                 }
4879         }
4880
4881         /*
4882          * Writing any value to intr-mbox-0 clears PCI INTA# and
4883          * chip-internal interrupt pending events.
4884          * Writing non-zero to intr-mbox-0 additional tells the
4885          * NIC to stop sending us irqs, engaging "in-intr-handler"
4886          * event coalescing.
4887          *
4888          * Flush the mailbox to de-assert the IRQ immediately to prevent
4889          * spurious interrupts.  The flush impacts performance but
4890          * excessive spurious interrupts can be worse in some cases.
4891          */
4892         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
4893         if (tg3_irq_sync(tp))
4894                 goto out;
4895         sblk->status &= ~SD_STATUS_UPDATED;
4896         if (likely(tg3_has_work(tnapi))) {
4897                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
4898                 napi_schedule(&tnapi->napi);
4899         } else {
4900                 /* No work, shared interrupt perhaps?  re-enable
4901                  * interrupts, and flush that PCI write
4902                  */
4903                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
4904                                0x00000000);
4905         }
4906 out:
4907         return IRQ_RETVAL(handled);
4908 }
4909
4910 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
4911 {
4912         struct tg3_napi *tnapi = dev_id;
4913         struct tg3 *tp = tnapi->tp;
4914         struct tg3_hw_status *sblk = tnapi->hw_status;
4915         unsigned int handled = 1;
4916
4917         /* In INTx mode, it is possible for the interrupt to arrive at
4918          * the CPU before the status block posted prior to the interrupt.
4919          * Reading the PCI State register will confirm whether the
4920          * interrupt is ours and will flush the status block.
4921          */
4922         if (unlikely(sblk->status_tag == tnapi->last_irq_tag)) {
4923                 if ((tp->tg3_flags & TG3_FLAG_CHIP_RESETTING) ||
4924                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
4925                         handled = 0;
4926                         goto out;
4927                 }
4928         }
4929
4930         /*
4931          * writing any value to intr-mbox-0 clears PCI INTA# and
4932          * chip-internal interrupt pending events.
4933          * writing non-zero to intr-mbox-0 additional tells the
4934          * NIC to stop sending us irqs, engaging "in-intr-handler"
4935          * event coalescing.
4936          *
4937          * Flush the mailbox to de-assert the IRQ immediately to prevent
4938          * spurious interrupts.  The flush impacts performance but
4939          * excessive spurious interrupts can be worse in some cases.
4940          */
4941         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
4942
4943         /*
4944          * In a shared interrupt configuration, sometimes other devices'
4945          * interrupts will scream.  We record the current status tag here
4946          * so that the above check can report that the screaming interrupts
4947          * are unhandled.  Eventually they will be silenced.
4948          */
4949         tnapi->last_irq_tag = sblk->status_tag;
4950
4951         if (tg3_irq_sync(tp))
4952                 goto out;
4953
4954         prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
4955
4956         napi_schedule(&tnapi->napi);
4957
4958 out:
4959         return IRQ_RETVAL(handled);
4960 }
4961
4962 /* ISR for interrupt test */
4963 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
4964 {
4965         struct tg3_napi *tnapi = dev_id;
4966         struct tg3 *tp = tnapi->tp;
4967         struct tg3_hw_status *sblk = tnapi->hw_status;
4968
4969         if ((sblk->status & SD_STATUS_UPDATED) ||
4970             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
4971                 tg3_disable_ints(tp);
4972                 return IRQ_RETVAL(1);
4973         }
4974         return IRQ_RETVAL(0);
4975 }
4976
4977 static int tg3_init_hw(struct tg3 *, int);
4978 static int tg3_halt(struct tg3 *, int, int);
4979
4980 /* Restart hardware after configuration changes, self-test, etc.
4981  * Invoked with tp->lock held.
4982  */
4983 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
4984         __releases(tp->lock)
4985         __acquires(tp->lock)
4986 {
4987         int err;
4988
4989         err = tg3_init_hw(tp, reset_phy);
4990         if (err) {
4991                 printk(KERN_ERR PFX "%s: Failed to re-initialize device, "
4992                        "aborting.\n", tp->dev->name);
4993                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
4994                 tg3_full_unlock(tp);
4995                 del_timer_sync(&tp->timer);
4996                 tp->irq_sync = 0;
4997                 tg3_napi_enable(tp);
4998                 dev_close(tp->dev);
4999                 tg3_full_lock(tp, 0);
5000         }
5001         return err;
5002 }
5003
5004 #ifdef CONFIG_NET_POLL_CONTROLLER
5005 static void tg3_poll_controller(struct net_device *dev)
5006 {
5007         int i;
5008         struct tg3 *tp = netdev_priv(dev);
5009
5010         for (i = 0; i < tp->irq_cnt; i++)
5011                 tg3_interrupt(tp->napi[i].irq_vec, dev);
5012 }
5013 #endif
5014
5015 static void tg3_reset_task(struct work_struct *work)
5016 {
5017         struct tg3 *tp = container_of(work, struct tg3, reset_task);
5018         int err;
5019         unsigned int restart_timer;
5020
5021         tg3_full_lock(tp, 0);
5022
5023         if (!netif_running(tp->dev)) {
5024                 tg3_full_unlock(tp);
5025                 return;
5026         }
5027
5028         tg3_full_unlock(tp);
5029
5030         tg3_phy_stop(tp);
5031
5032         tg3_netif_stop(tp);
5033
5034         tg3_full_lock(tp, 1);
5035
5036         restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
5037         tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;
5038
5039         if (tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING) {
5040                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
5041                 tp->write32_rx_mbox = tg3_write_flush_reg32;
5042                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
5043                 tp->tg3_flags &= ~TG3_FLAG_TX_RECOVERY_PENDING;
5044         }
5045
5046         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
5047         err = tg3_init_hw(tp, 1);
5048         if (err)
5049                 goto out;
5050
5051         tg3_netif_start(tp);
5052
5053         if (restart_timer)
5054                 mod_timer(&tp->timer, jiffies + 1);
5055
5056 out:
5057         tg3_full_unlock(tp);
5058
5059         if (!err)
5060                 tg3_phy_start(tp);
5061 }
5062
5063 static void tg3_dump_short_state(struct tg3 *tp)
5064 {
5065         printk(KERN_ERR PFX "DEBUG: MAC_TX_STATUS[%08x] MAC_RX_STATUS[%08x]\n",
5066                tr32(MAC_TX_STATUS), tr32(MAC_RX_STATUS));
5067         printk(KERN_ERR PFX "DEBUG: RDMAC_STATUS[%08x] WDMAC_STATUS[%08x]\n",
5068                tr32(RDMAC_STATUS), tr32(WDMAC_STATUS));
5069 }
5070
5071 static void tg3_tx_timeout(struct net_device *dev)
5072 {
5073         struct tg3 *tp = netdev_priv(dev);
5074
5075         if (netif_msg_tx_err(tp)) {
5076                 printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
5077                        dev->name);
5078                 tg3_dump_short_state(tp);
5079         }
5080
5081         schedule_work(&tp->reset_task);
5082 }
5083
5084 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
5085 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
5086 {
5087         u32 base = (u32) mapping & 0xffffffff;
5088
5089         return ((base > 0xffffdcc0) &&
5090                 (base + len + 8 < base));
5091 }
5092
5093 /* Test for DMA addresses > 40-bit */
5094 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
5095                                           int len)
5096 {
5097 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
5098         if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG)
5099                 return (((u64) mapping + len) > DMA_BIT_MASK(40));
5100         return 0;
5101 #else
5102         return 0;
5103 #endif
5104 }
5105
5106 static void tg3_set_txd(struct tg3_napi *, int, dma_addr_t, int, u32, u32);
5107
5108 /* Workaround 4GB and 40-bit hardware DMA bugs. */
5109 static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
5110                                        u32 last_plus_one, u32 *start,
5111                                        u32 base_flags, u32 mss)
5112 {
5113         struct tg3_napi *tnapi = &tp->napi[0];
5114         struct sk_buff *new_skb;
5115         dma_addr_t new_addr = 0;
5116         u32 entry = *start;
5117         int i, ret = 0;
5118
5119         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
5120                 new_skb = skb_copy(skb, GFP_ATOMIC);
5121         else {
5122                 int more_headroom = 4 - ((unsigned long)skb->data & 3);
5123
5124                 new_skb = skb_copy_expand(skb,
5125                                           skb_headroom(skb) + more_headroom,
5126                                           skb_tailroom(skb), GFP_ATOMIC);
5127         }
5128
5129         if (!new_skb) {
5130                 ret = -1;
5131         } else {
5132                 /* New SKB is guaranteed to be linear. */
5133                 entry = *start;
5134                 ret = skb_dma_map(&tp->pdev->dev, new_skb, DMA_TO_DEVICE);
5135                 new_addr = skb_shinfo(new_skb)->dma_head;
5136
5137                 /* Make sure new skb does not cross any 4G boundaries.
5138                  * Drop the packet if it does.
5139                  */
5140                 if (ret || tg3_4g_overflow_test(new_addr, new_skb->len)) {
5141                         if (!ret)
5142                                 skb_dma_unmap(&tp->pdev->dev, new_skb,
5143                                               DMA_TO_DEVICE);
5144                         ret = -1;
5145                         dev_kfree_skb(new_skb);
5146                         new_skb = NULL;
5147                 } else {
5148                         tg3_set_txd(tnapi, entry, new_addr, new_skb->len,
5149                                     base_flags, 1 | (mss << 1));
5150                         *start = NEXT_TX(entry);
5151                 }
5152         }
5153
5154         /* Now clean up the sw ring entries. */
5155         i = 0;
5156         while (entry != last_plus_one) {
5157                 if (i == 0)
5158                         tnapi->tx_buffers[entry].skb = new_skb;
5159                 else
5160                         tnapi->tx_buffers[entry].skb = NULL;
5161                 entry = NEXT_TX(entry);
5162                 i++;
5163         }
5164
5165         skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE);
5166         dev_kfree_skb(skb);
5167
5168         return ret;
5169 }
5170
5171 static void tg3_set_txd(struct tg3_napi *tnapi, int entry,
5172                         dma_addr_t mapping, int len, u32 flags,
5173                         u32 mss_and_is_end)
5174 {
5175         struct tg3_tx_buffer_desc *txd = &tnapi->tx_ring[entry];
5176         int is_end = (mss_and_is_end & 0x1);
5177         u32 mss = (mss_and_is_end >> 1);
5178         u32 vlan_tag = 0;
5179
5180         if (is_end)
5181                 flags |= TXD_FLAG_END;
5182         if (flags & TXD_FLAG_VLAN) {
5183                 vlan_tag = flags >> 16;
5184                 flags &= 0xffff;
5185         }
5186         vlan_tag |= (mss << TXD_MSS_SHIFT);
5187
5188         txd->addr_hi = ((u64) mapping >> 32);
5189         txd->addr_lo = ((u64) mapping & 0xffffffff);
5190         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
5191         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
5192 }
5193
5194 /* hard_start_xmit for devices that don't have any bugs and
5195  * support TG3_FLG2_HW_TSO_2 only.
5196  */
5197 static netdev_tx_t tg3_start_xmit(struct sk_buff *skb,
5198                                   struct net_device *dev)
5199 {
5200         struct tg3 *tp = netdev_priv(dev);
5201         u32 len, entry, base_flags, mss;
5202         struct skb_shared_info *sp;
5203         dma_addr_t mapping;
5204         struct tg3_napi *tnapi;
5205         struct netdev_queue *txq;
5206
5207         txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
5208         tnapi = &tp->napi[skb_get_queue_mapping(skb)];
5209         if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX)
5210                 tnapi++;
5211
5212         /* We are running in BH disabled context with netif_tx_lock
5213          * and TX reclaim runs via tp->napi.poll inside of a software
5214          * interrupt.  Furthermore, IRQ processing runs lockless so we have
5215          * no IRQ context deadlocks to worry about either.  Rejoice!
5216          */
5217         if (unlikely(tg3_tx_avail(tnapi) <= (skb_shinfo(skb)->nr_frags + 1))) {
5218                 if (!netif_tx_queue_stopped(txq)) {
5219                         netif_tx_stop_queue(txq);
5220
5221                         /* This is a hard error, log it. */
5222                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
5223                                "queue awake!\n", dev->name);
5224                 }
5225                 return NETDEV_TX_BUSY;
5226         }
5227
5228         entry = tnapi->tx_prod;
5229         base_flags = 0;
5230         mss = 0;
5231         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
5232                 int tcp_opt_len, ip_tcp_len;
5233                 u32 hdrlen;
5234
5235                 if (skb_header_cloned(skb) &&
5236                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
5237                         dev_kfree_skb(skb);
5238                         goto out_unlock;
5239                 }
5240
5241                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
5242                         hdrlen = skb_headlen(skb) - ETH_HLEN;
5243                 else {
5244                         struct iphdr *iph = ip_hdr(skb);
5245
5246                         tcp_opt_len = tcp_optlen(skb);
5247                         ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
5248
5249                         iph->check = 0;
5250                         iph->tot_len = htons(mss + ip_tcp_len + tcp_opt_len);
5251                         hdrlen = ip_tcp_len + tcp_opt_len;
5252                 }
5253
5254                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
5255                         mss |= (hdrlen & 0xc) << 12;
5256                         if (hdrlen & 0x10)
5257                                 base_flags |= 0x00000010;
5258                         base_flags |= (hdrlen & 0x3e0) << 5;
5259                 } else
5260                         mss |= hdrlen << 9;
5261
5262                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
5263                                TXD_FLAG_CPU_POST_DMA);
5264
5265                 tcp_hdr(skb)->check = 0;
5266
5267         }
5268         else if (skb->ip_summed == CHECKSUM_PARTIAL)
5269                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
5270 #if TG3_VLAN_TAG_USED
5271         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
5272                 base_flags |= (TXD_FLAG_VLAN |
5273                                (vlan_tx_tag_get(skb) << 16));
5274 #endif
5275
5276         if (skb_dma_map(&tp->pdev->dev, skb, DMA_TO_DEVICE)) {
5277                 dev_kfree_skb(skb);
5278                 goto out_unlock;
5279         }
5280
5281         sp = skb_shinfo(skb);
5282
5283         mapping = sp->dma_head;
5284
5285         tnapi->tx_buffers[entry].skb = skb;
5286
5287         len = skb_headlen(skb);
5288
5289         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 &&
5290             !mss && skb->len > ETH_DATA_LEN)
5291                 base_flags |= TXD_FLAG_JMB_PKT;
5292
5293         tg3_set_txd(tnapi, entry, mapping, len, base_flags,
5294                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
5295
5296         entry = NEXT_TX(entry);
5297
5298         /* Now loop through additional data fragments, and queue them. */
5299         if (skb_shinfo(skb)->nr_frags > 0) {
5300                 unsigned int i, last;
5301
5302                 last = skb_shinfo(skb)->nr_frags - 1;
5303                 for (i = 0; i <= last; i++) {
5304                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5305
5306                         len = frag->size;
5307                         mapping = sp->dma_maps[i];
5308                         tnapi->tx_buffers[entry].skb = NULL;
5309
5310                         tg3_set_txd(tnapi, entry, mapping, len,
5311                                     base_flags, (i == last) | (mss << 1));
5312
5313                         entry = NEXT_TX(entry);
5314                 }
5315         }
5316
5317         /* Packets are ready, update Tx producer idx local and on card. */
5318         tw32_tx_mbox(tnapi->prodmbox, entry);
5319
5320         tnapi->tx_prod = entry;
5321         if (unlikely(tg3_tx_avail(tnapi) <= (MAX_SKB_FRAGS + 1))) {
5322                 netif_tx_stop_queue(txq);
5323                 if (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi))
5324                         netif_tx_wake_queue(txq);
5325         }
5326
5327 out_unlock:
5328         mmiowb();
5329
5330         return NETDEV_TX_OK;
5331 }
5332
5333 static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *,
5334                                           struct net_device *);
5335
5336 /* Use GSO to workaround a rare TSO bug that may be triggered when the
5337  * TSO header is greater than 80 bytes.
5338  */
5339 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
5340 {
5341         struct sk_buff *segs, *nskb;
5342         u32 frag_cnt_est = skb_shinfo(skb)->gso_segs * 3;
5343
5344         /* Estimate the number of fragments in the worst case */
5345         if (unlikely(tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)) {
5346                 netif_stop_queue(tp->dev);
5347                 if (tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)
5348                         return NETDEV_TX_BUSY;
5349
5350                 netif_wake_queue(tp->dev);
5351         }
5352
5353         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
5354         if (IS_ERR(segs))
5355                 goto tg3_tso_bug_end;
5356
5357         do {
5358                 nskb = segs;
5359                 segs = segs->next;
5360                 nskb->next = NULL;
5361                 tg3_start_xmit_dma_bug(nskb, tp->dev);
5362         } while (segs);
5363
5364 tg3_tso_bug_end:
5365         dev_kfree_skb(skb);
5366
5367         return NETDEV_TX_OK;
5368 }
5369
5370 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
5371  * support TG3_FLG2_HW_TSO_1 or firmware TSO only.
5372  */
5373 static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb,
5374                                           struct net_device *dev)
5375 {
5376         struct tg3 *tp = netdev_priv(dev);
5377         u32 len, entry, base_flags, mss;
5378         struct skb_shared_info *sp;
5379         int would_hit_hwbug;
5380         dma_addr_t mapping;
5381         struct tg3_napi *tnapi = &tp->napi[0];
5382
5383         len = skb_headlen(skb);
5384
5385         /* We are running in BH disabled context with netif_tx_lock
5386          * and TX reclaim runs via tp->napi.poll inside of a software
5387          * interrupt.  Furthermore, IRQ processing runs lockless so we have
5388          * no IRQ context deadlocks to worry about either.  Rejoice!
5389          */
5390         if (unlikely(tg3_tx_avail(tnapi) <= (skb_shinfo(skb)->nr_frags + 1))) {
5391                 if (!netif_queue_stopped(dev)) {
5392                         netif_stop_queue(dev);
5393
5394                         /* This is a hard error, log it. */
5395                         printk(KERN_ERR PFX "%s: BUG! Tx Ring full when "
5396                                "queue awake!\n", dev->name);
5397                 }
5398                 return NETDEV_TX_BUSY;
5399         }
5400
5401         entry = tnapi->tx_prod;
5402         base_flags = 0;
5403         if (skb->ip_summed == CHECKSUM_PARTIAL)
5404                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
5405         mss = 0;
5406         if ((mss = skb_shinfo(skb)->gso_size) != 0) {
5407                 struct iphdr *iph;
5408                 int tcp_opt_len, ip_tcp_len, hdr_len;
5409
5410                 if (skb_header_cloned(skb) &&
5411                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
5412                         dev_kfree_skb(skb);
5413                         goto out_unlock;
5414                 }
5415
5416                 tcp_opt_len = tcp_optlen(skb);
5417                 ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
5418
5419                 hdr_len = ip_tcp_len + tcp_opt_len;
5420                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
5421                              (tp->tg3_flags2 & TG3_FLG2_TSO_BUG))
5422                         return (tg3_tso_bug(tp, skb));
5423
5424                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
5425                                TXD_FLAG_CPU_POST_DMA);
5426
5427                 iph = ip_hdr(skb);
5428                 iph->check = 0;
5429                 iph->tot_len = htons(mss + hdr_len);
5430                 if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
5431                         tcp_hdr(skb)->check = 0;
5432                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
5433                 } else
5434                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
5435                                                                  iph->daddr, 0,
5436                                                                  IPPROTO_TCP,
5437                                                                  0);
5438
5439                 if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) ||
5440                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) {
5441                         if (tcp_opt_len || iph->ihl > 5) {
5442                                 int tsflags;
5443
5444                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5445                                 mss |= (tsflags << 11);
5446                         }
5447                 } else {
5448                         if (tcp_opt_len || iph->ihl > 5) {
5449                                 int tsflags;
5450
5451                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5452                                 base_flags |= tsflags << 12;
5453                         }
5454                 }
5455         }
5456 #if TG3_VLAN_TAG_USED
5457         if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
5458                 base_flags |= (TXD_FLAG_VLAN |
5459                                (vlan_tx_tag_get(skb) << 16));
5460 #endif
5461
5462         if (skb_dma_map(&tp->pdev->dev, skb, DMA_TO_DEVICE)) {
5463                 dev_kfree_skb(skb);
5464                 goto out_unlock;
5465         }
5466
5467         sp = skb_shinfo(skb);
5468
5469         mapping = sp->dma_head;
5470
5471         tnapi->tx_buffers[entry].skb = skb;
5472
5473         would_hit_hwbug = 0;
5474
5475         if (tp->tg3_flags3 & TG3_FLG3_5701_DMA_BUG)
5476                 would_hit_hwbug = 1;
5477         else if (tg3_4g_overflow_test(mapping, len))
5478                 would_hit_hwbug = 1;
5479
5480         tg3_set_txd(tnapi, entry, mapping, len, base_flags,
5481                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
5482
5483         entry = NEXT_TX(entry);
5484
5485         /* Now loop through additional data fragments, and queue them. */
5486         if (skb_shinfo(skb)->nr_frags > 0) {
5487                 unsigned int i, last;
5488
5489                 last = skb_shinfo(skb)->nr_frags - 1;
5490                 for (i = 0; i <= last; i++) {
5491                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5492
5493                         len = frag->size;
5494                         mapping = sp->dma_maps[i];
5495
5496                         tnapi->tx_buffers[entry].skb = NULL;
5497
5498                         if (tg3_4g_overflow_test(mapping, len))
5499                                 would_hit_hwbug = 1;
5500
5501                         if (tg3_40bit_overflow_test(tp, mapping, len))
5502                                 would_hit_hwbug = 1;
5503
5504                         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
5505                                 tg3_set_txd(tnapi, entry, mapping, len,
5506                                             base_flags, (i == last)|(mss << 1));
5507                         else
5508                                 tg3_set_txd(tnapi, entry, mapping, len,
5509                                             base_flags, (i == last));
5510
5511                         entry = NEXT_TX(entry);
5512                 }
5513         }
5514
5515         if (would_hit_hwbug) {
5516                 u32 last_plus_one = entry;
5517                 u32 start;
5518
5519                 start = entry - 1 - skb_shinfo(skb)->nr_frags;
5520                 start &= (TG3_TX_RING_SIZE - 1);
5521
5522                 /* If the workaround fails due to memory/mapping
5523                  * failure, silently drop this packet.
5524                  */
5525                 if (tigon3_dma_hwbug_workaround(tp, skb, last_plus_one,
5526                                                 &start, base_flags, mss))
5527                         goto out_unlock;
5528
5529                 entry = start;
5530         }
5531
5532         /* Packets are ready, update Tx producer idx local and on card. */
5533         tw32_tx_mbox(MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW, entry);
5534
5535         tnapi->tx_prod = entry;
5536         if (unlikely(tg3_tx_avail(tnapi) <= (MAX_SKB_FRAGS + 1))) {
5537                 netif_stop_queue(dev);
5538                 if (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi))
5539                         netif_wake_queue(tp->dev);
5540         }
5541
5542 out_unlock:
5543         mmiowb();
5544
5545         return NETDEV_TX_OK;
5546 }
5547
5548 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
5549                                int new_mtu)
5550 {
5551         dev->mtu = new_mtu;
5552
5553         if (new_mtu > ETH_DATA_LEN) {
5554                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
5555                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
5556                         ethtool_op_set_tso(dev, 0);
5557                 }
5558                 else
5559                         tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
5560         } else {
5561                 if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
5562                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
5563                 tp->tg3_flags &= ~TG3_FLAG_JUMBO_RING_ENABLE;
5564         }
5565 }
5566
5567 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
5568 {
5569         struct tg3 *tp = netdev_priv(dev);
5570         int err;
5571
5572         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
5573                 return -EINVAL;
5574
5575         if (!netif_running(dev)) {
5576                 /* We'll just catch it later when the
5577                  * device is up'd.
5578                  */
5579                 tg3_set_mtu(dev, tp, new_mtu);
5580                 return 0;
5581         }
5582
5583         tg3_phy_stop(tp);
5584
5585         tg3_netif_stop(tp);
5586
5587         tg3_full_lock(tp, 1);
5588
5589         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
5590
5591         tg3_set_mtu(dev, tp, new_mtu);
5592
5593         err = tg3_restart_hw(tp, 0);
5594
5595         if (!err)
5596                 tg3_netif_start(tp);
5597
5598         tg3_full_unlock(tp);
5599
5600         if (!err)
5601                 tg3_phy_start(tp);
5602
5603         return err;
5604 }
5605
5606 static void tg3_rx_prodring_free(struct tg3 *tp,
5607                                  struct tg3_rx_prodring_set *tpr)
5608 {
5609         int i;
5610         struct ring_info *rxp;
5611
5612         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
5613                 rxp = &tpr->rx_std_buffers[i];
5614
5615                 if (rxp->skb == NULL)
5616                         continue;
5617
5618                 pci_unmap_single(tp->pdev,
5619                                  pci_unmap_addr(rxp, mapping),
5620                                  tp->rx_pkt_map_sz,
5621                                  PCI_DMA_FROMDEVICE);
5622                 dev_kfree_skb_any(rxp->skb);
5623                 rxp->skb = NULL;
5624         }
5625
5626         if (tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) {
5627                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
5628                         rxp = &tpr->rx_jmb_buffers[i];
5629
5630                         if (rxp->skb == NULL)
5631                                 continue;
5632
5633                         pci_unmap_single(tp->pdev,
5634                                          pci_unmap_addr(rxp, mapping),
5635                                          TG3_RX_JMB_MAP_SZ,
5636                                          PCI_DMA_FROMDEVICE);
5637                         dev_kfree_skb_any(rxp->skb);
5638                         rxp->skb = NULL;
5639                 }
5640         }
5641 }
5642
5643 /* Initialize tx/rx rings for packet processing.
5644  *
5645  * The chip has been shut down and the driver detached from
5646  * the networking, so no interrupts or new tx packets will
5647  * end up in the driver.  tp->{tx,}lock are held and thus
5648  * we may not sleep.
5649  */
5650 static int tg3_rx_prodring_alloc(struct tg3 *tp,
5651                                  struct tg3_rx_prodring_set *tpr)
5652 {
5653         u32 i, rx_pkt_dma_sz;
5654         struct tg3_napi *tnapi = &tp->napi[0];
5655
5656         /* Zero out all descriptors. */
5657         memset(tpr->rx_std, 0, TG3_RX_RING_BYTES);
5658
5659         rx_pkt_dma_sz = TG3_RX_STD_DMA_SZ;
5660         if ((tp->tg3_flags2 & TG3_FLG2_5780_CLASS) &&
5661             tp->dev->mtu > ETH_DATA_LEN)
5662                 rx_pkt_dma_sz = TG3_RX_JMB_DMA_SZ;
5663         tp->rx_pkt_map_sz = TG3_RX_DMA_TO_MAP_SZ(rx_pkt_dma_sz);
5664
5665         /* Initialize invariants of the rings, we only set this
5666          * stuff once.  This works because the card does not
5667          * write into the rx buffer posting rings.
5668          */
5669         for (i = 0; i < TG3_RX_RING_SIZE; i++) {
5670                 struct tg3_rx_buffer_desc *rxd;
5671
5672                 rxd = &tpr->rx_std[i];
5673                 rxd->idx_len = rx_pkt_dma_sz << RXD_LEN_SHIFT;
5674                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
5675                 rxd->opaque = (RXD_OPAQUE_RING_STD |
5676                                (i << RXD_OPAQUE_INDEX_SHIFT));
5677         }
5678
5679         /* Now allocate fresh SKBs for each rx ring. */
5680         for (i = 0; i < tp->rx_pending; i++) {
5681                 if (tg3_alloc_rx_skb(tnapi, RXD_OPAQUE_RING_STD, -1, i) < 0) {
5682                         printk(KERN_WARNING PFX
5683                                "%s: Using a smaller RX standard ring, "
5684                                "only %d out of %d buffers were allocated "
5685                                "successfully.\n",
5686                                tp->dev->name, i, tp->rx_pending);
5687                         if (i == 0)
5688                                 goto initfail;
5689                         tp->rx_pending = i;
5690                         break;
5691                 }
5692         }
5693
5694         if (!(tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE))
5695                 goto done;
5696
5697         memset(tpr->rx_jmb, 0, TG3_RX_JUMBO_RING_BYTES);
5698
5699         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
5700                 for (i = 0; i < TG3_RX_JUMBO_RING_SIZE; i++) {
5701                         struct tg3_rx_buffer_desc *rxd;
5702
5703                         rxd = &tpr->rx_jmb[i].std;
5704                         rxd->idx_len = TG3_RX_JMB_DMA_SZ << RXD_LEN_SHIFT;
5705                         rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
5706                                 RXD_FLAG_JUMBO;
5707                         rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
5708                                (i << RXD_OPAQUE_INDEX_SHIFT));
5709                 }
5710
5711                 for (i = 0; i < tp->rx_jumbo_pending; i++) {
5712                         if (tg3_alloc_rx_skb(tnapi, RXD_OPAQUE_RING_JUMBO,
5713                                              -1, i) < 0) {
5714                                 printk(KERN_WARNING PFX
5715                                        "%s: Using a smaller RX jumbo ring, "
5716                                        "only %d out of %d buffers were "
5717                                        "allocated successfully.\n",
5718                                        tp->dev->name, i, tp->rx_jumbo_pending);
5719                                 if (i == 0)
5720                                         goto initfail;
5721                                 tp->rx_jumbo_pending = i;
5722                                 break;
5723                         }
5724                 }
5725         }
5726
5727 done:
5728         return 0;
5729
5730 initfail:
5731         tg3_rx_prodring_free(tp, tpr);
5732         return -ENOMEM;
5733 }
5734
5735 static void tg3_rx_prodring_fini(struct tg3 *tp,
5736                                  struct tg3_rx_prodring_set *tpr)
5737 {
5738         kfree(tpr->rx_std_buffers);
5739         tpr->rx_std_buffers = NULL;
5740         kfree(tpr->rx_jmb_buffers);
5741         tpr->rx_jmb_buffers = NULL;
5742         if (tpr->rx_std) {
5743                 pci_free_consistent(tp->pdev, TG3_RX_RING_BYTES,
5744                                     tpr->rx_std, tpr->rx_std_mapping);
5745                 tpr->rx_std = NULL;
5746         }
5747         if (tpr->rx_jmb) {
5748                 pci_free_consistent(tp->pdev, TG3_RX_JUMBO_RING_BYTES,
5749                                     tpr->rx_jmb, tpr->rx_jmb_mapping);
5750                 tpr->rx_jmb = NULL;
5751         }
5752 }
5753
5754 static int tg3_rx_prodring_init(struct tg3 *tp,
5755                                 struct tg3_rx_prodring_set *tpr)
5756 {
5757         tpr->rx_std_buffers = kzalloc(sizeof(struct ring_info) *
5758                                       TG3_RX_RING_SIZE, GFP_KERNEL);
5759         if (!tpr->rx_std_buffers)
5760                 return -ENOMEM;
5761
5762         tpr->rx_std = pci_alloc_consistent(tp->pdev, TG3_RX_RING_BYTES,
5763                                            &tpr->rx_std_mapping);
5764         if (!tpr->rx_std)
5765                 goto err_out;
5766
5767         if (tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) {
5768                 tpr->rx_jmb_buffers = kzalloc(sizeof(struct ring_info) *
5769                                               TG3_RX_JUMBO_RING_SIZE,
5770                                               GFP_KERNEL);
5771                 if (!tpr->rx_jmb_buffers)
5772                         goto err_out;
5773
5774                 tpr->rx_jmb = pci_alloc_consistent(tp->pdev,
5775                                                    TG3_RX_JUMBO_RING_BYTES,
5776                                                    &tpr->rx_jmb_mapping);
5777                 if (!tpr->rx_jmb)
5778                         goto err_out;
5779         }
5780
5781         return 0;
5782
5783 err_out:
5784         tg3_rx_prodring_fini(tp, tpr);
5785         return -ENOMEM;
5786 }
5787
5788 /* Free up pending packets in all rx/tx rings.
5789  *
5790  * The chip has been shut down and the driver detached from
5791  * the networking, so no interrupts or new tx packets will
5792  * end up in the driver.  tp->{tx,}lock is not held and we are not
5793  * in an interrupt context and thus may sleep.
5794  */
5795 static void tg3_free_rings(struct tg3 *tp)
5796 {
5797         int i, j;
5798
5799         for (j = 0; j < tp->irq_cnt; j++) {
5800                 struct tg3_napi *tnapi = &tp->napi[j];
5801
5802                 if (!tnapi->tx_buffers)
5803                         continue;
5804
5805                 for (i = 0; i < TG3_TX_RING_SIZE; ) {
5806                         struct tx_ring_info *txp;
5807                         struct sk_buff *skb;
5808
5809                         txp = &tnapi->tx_buffers[i];
5810                         skb = txp->skb;
5811
5812                         if (skb == NULL) {
5813                                 i++;
5814                                 continue;
5815                         }
5816
5817                         skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE);
5818
5819                         txp->skb = NULL;
5820
5821                         i += skb_shinfo(skb)->nr_frags + 1;
5822
5823                         dev_kfree_skb_any(skb);
5824                 }
5825         }
5826
5827         tg3_rx_prodring_free(tp, &tp->prodring[0]);
5828 }
5829
5830 /* Initialize tx/rx rings for packet processing.
5831  *
5832  * The chip has been shut down and the driver detached from
5833  * the networking, so no interrupts or new tx packets will
5834  * end up in the driver.  tp->{tx,}lock are held and thus
5835  * we may not sleep.
5836  */
5837 static int tg3_init_rings(struct tg3 *tp)
5838 {
5839         int i;
5840
5841         /* Free up all the SKBs. */
5842         tg3_free_rings(tp);
5843
5844         for (i = 0; i < tp->irq_cnt; i++) {
5845                 struct tg3_napi *tnapi = &tp->napi[i];
5846
5847                 tnapi->last_tag = 0;
5848                 tnapi->last_irq_tag = 0;
5849                 tnapi->hw_status->status = 0;
5850                 tnapi->hw_status->status_tag = 0;
5851                 memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
5852
5853                 tnapi->tx_prod = 0;
5854                 tnapi->tx_cons = 0;
5855                 if (tnapi->tx_ring)
5856                         memset(tnapi->tx_ring, 0, TG3_TX_RING_BYTES);
5857
5858                 tnapi->rx_rcb_ptr = 0;
5859                 if (tnapi->rx_rcb)
5860                         memset(tnapi->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
5861         }
5862
5863         return tg3_rx_prodring_alloc(tp, &tp->prodring[0]);
5864 }
5865
5866 /*
5867  * Must not be invoked with interrupt sources disabled and
5868  * the hardware shutdown down.
5869  */
5870 static void tg3_free_consistent(struct tg3 *tp)
5871 {
5872         int i;
5873
5874         for (i = 0; i < tp->irq_cnt; i++) {
5875                 struct tg3_napi *tnapi = &tp->napi[i];
5876
5877                 if (tnapi->tx_ring) {
5878                         pci_free_consistent(tp->pdev, TG3_TX_RING_BYTES,
5879                                 tnapi->tx_ring, tnapi->tx_desc_mapping);
5880                         tnapi->tx_ring = NULL;
5881                 }
5882
5883                 kfree(tnapi->tx_buffers);
5884                 tnapi->tx_buffers = NULL;
5885
5886                 if (tnapi->rx_rcb) {
5887                         pci_free_consistent(tp->pdev, TG3_RX_RCB_RING_BYTES(tp),
5888                                             tnapi->rx_rcb,
5889                                             tnapi->rx_rcb_mapping);
5890                         tnapi->rx_rcb = NULL;
5891                 }
5892
5893                 if (tnapi->hw_status) {
5894                         pci_free_consistent(tp->pdev, TG3_HW_STATUS_SIZE,
5895                                             tnapi->hw_status,
5896                                             tnapi->status_mapping);
5897                         tnapi->hw_status = NULL;
5898                 }
5899         }
5900
5901         if (tp->hw_stats) {
5902                 pci_free_consistent(tp->pdev, sizeof(struct tg3_hw_stats),
5903                                     tp->hw_stats, tp->stats_mapping);
5904                 tp->hw_stats = NULL;
5905         }
5906
5907         tg3_rx_prodring_fini(tp, &tp->prodring[0]);
5908 }
5909
5910 /*
5911  * Must not be invoked with interrupt sources disabled and
5912  * the hardware shutdown down.  Can sleep.
5913  */
5914 static int tg3_alloc_consistent(struct tg3 *tp)
5915 {
5916         int i;
5917
5918         if (tg3_rx_prodring_init(tp, &tp->prodring[0]))
5919                 return -ENOMEM;
5920
5921         tp->hw_stats = pci_alloc_consistent(tp->pdev,
5922                                             sizeof(struct tg3_hw_stats),
5923                                             &tp->stats_mapping);
5924         if (!tp->hw_stats)
5925                 goto err_out;
5926
5927         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
5928
5929         for (i = 0; i < tp->irq_cnt; i++) {
5930                 struct tg3_napi *tnapi = &tp->napi[i];
5931                 struct tg3_hw_status *sblk;
5932
5933                 tnapi->hw_status = pci_alloc_consistent(tp->pdev,
5934                                                         TG3_HW_STATUS_SIZE,
5935                                                         &tnapi->status_mapping);
5936                 if (!tnapi->hw_status)
5937                         goto err_out;
5938
5939                 memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
5940                 sblk = tnapi->hw_status;
5941
5942                 /*
5943                  * When RSS is enabled, the status block format changes
5944                  * slightly.  The "rx_jumbo_consumer", "reserved",
5945                  * and "rx_mini_consumer" members get mapped to the
5946                  * other three rx return ring producer indexes.
5947                  */
5948                 switch (i) {
5949                 default:
5950                         tnapi->rx_rcb_prod_idx = &sblk->idx[0].rx_producer;
5951                         break;
5952                 case 2:
5953                         tnapi->rx_rcb_prod_idx = &sblk->rx_jumbo_consumer;
5954                         break;
5955                 case 3:
5956                         tnapi->rx_rcb_prod_idx = &sblk->reserved;
5957                         break;
5958                 case 4:
5959                         tnapi->rx_rcb_prod_idx = &sblk->rx_mini_consumer;
5960                         break;
5961                 }
5962
5963                 /*
5964                  * If multivector RSS is enabled, vector 0 does not handle
5965                  * rx or tx interrupts.  Don't allocate any resources for it.
5966                  */
5967                 if (!i && (tp->tg3_flags3 & TG3_FLG3_ENABLE_RSS))
5968                         continue;
5969
5970                 tnapi->rx_rcb = pci_alloc_consistent(tp->pdev,
5971                                                      TG3_RX_RCB_RING_BYTES(tp),
5972                                                      &tnapi->rx_rcb_mapping);
5973                 if (!tnapi->rx_rcb)
5974                         goto err_out;
5975
5976                 memset(tnapi->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
5977
5978                 tnapi->tx_buffers = kzalloc(sizeof(struct tx_ring_info) *
5979                                             TG3_TX_RING_SIZE, GFP_KERNEL);
5980                 if (!tnapi->tx_buffers)
5981                         goto err_out;
5982
5983                 tnapi->tx_ring = pci_alloc_consistent(tp->pdev,
5984                                                       TG3_TX_RING_BYTES,
5985                                                       &tnapi->tx_desc_mapping);
5986                 if (!tnapi->tx_ring)
5987                         goto err_out;
5988         }
5989
5990         return 0;
5991
5992 err_out:
5993         tg3_free_consistent(tp);
5994         return -ENOMEM;
5995 }
5996
5997 #define MAX_WAIT_CNT 1000
5998
5999 /* To stop a block, clear the enable bit and poll till it
6000  * clears.  tp->lock is held.
6001  */
6002 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
6003 {
6004         unsigned int i;
6005         u32 val;
6006
6007         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
6008                 switch (ofs) {
6009                 case RCVLSC_MODE:
6010                 case DMAC_MODE:
6011                 case MBFREE_MODE:
6012                 case BUFMGR_MODE:
6013                 case MEMARB_MODE:
6014                         /* We can't enable/disable these bits of the
6015                          * 5705/5750, just say success.
6016                          */
6017                         return 0;
6018
6019                 default:
6020                         break;
6021                 }
6022         }
6023
6024         val = tr32(ofs);
6025         val &= ~enable_bit;
6026         tw32_f(ofs, val);
6027
6028         for (i = 0; i < MAX_WAIT_CNT; i++) {
6029                 udelay(100);
6030                 val = tr32(ofs);
6031                 if ((val & enable_bit) == 0)
6032                         break;
6033         }
6034
6035         if (i == MAX_WAIT_CNT && !silent) {
6036                 printk(KERN_ERR PFX "tg3_stop_block timed out, "
6037                        "ofs=%lx enable_bit=%x\n",
6038                        ofs, enable_bit);
6039                 return -ENODEV;
6040         }
6041
6042         return 0;
6043 }
6044
6045 /* tp->lock is held. */
6046 static int tg3_abort_hw(struct tg3 *tp, int silent)
6047 {
6048         int i, err;
6049
6050         tg3_disable_ints(tp);
6051
6052         tp->rx_mode &= ~RX_MODE_ENABLE;
6053         tw32_f(MAC_RX_MODE, tp->rx_mode);
6054         udelay(10);
6055
6056         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
6057         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
6058         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
6059         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
6060         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
6061         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
6062
6063         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
6064         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
6065         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
6066         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
6067         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
6068         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
6069         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
6070
6071         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
6072         tw32_f(MAC_MODE, tp->mac_mode);
6073         udelay(40);
6074
6075         tp->tx_mode &= ~TX_MODE_ENABLE;
6076         tw32_f(MAC_TX_MODE, tp->tx_mode);
6077
6078         for (i = 0; i < MAX_WAIT_CNT; i++) {
6079                 udelay(100);
6080                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
6081                         break;
6082         }
6083         if (i >= MAX_WAIT_CNT) {
6084                 printk(KERN_ERR PFX "tg3_abort_hw timed out for %s, "
6085                        "TX_MODE_ENABLE will not clear MAC_TX_MODE=%08x\n",
6086                        tp->dev->name, tr32(MAC_TX_MODE));
6087                 err |= -ENODEV;
6088         }
6089
6090         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
6091         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
6092         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
6093
6094         tw32(FTQ_RESET, 0xffffffff);
6095         tw32(FTQ_RESET, 0x00000000);
6096
6097         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
6098         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
6099
6100         for (i = 0; i < tp->irq_cnt; i++) {
6101                 struct tg3_napi *tnapi = &tp->napi[i];
6102                 if (tnapi->hw_status)
6103                         memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
6104         }
6105         if (tp->hw_stats)
6106                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
6107
6108         return err;
6109 }
6110
6111 static void tg3_ape_send_event(struct tg3 *tp, u32 event)
6112 {
6113         int i;
6114         u32 apedata;
6115
6116         apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
6117         if (apedata != APE_SEG_SIG_MAGIC)
6118                 return;
6119
6120         apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
6121         if (!(apedata & APE_FW_STATUS_READY))
6122                 return;
6123
6124         /* Wait for up to 1 millisecond for APE to service previous event. */
6125         for (i = 0; i < 10; i++) {
6126                 if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
6127                         return;
6128
6129                 apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
6130
6131                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
6132                         tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
6133                                         event | APE_EVENT_STATUS_EVENT_PENDING);
6134
6135                 tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
6136
6137                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
6138                         break;
6139
6140                 udelay(100);
6141         }
6142
6143         if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
6144                 tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
6145 }
6146
6147 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
6148 {
6149         u32 event;
6150         u32 apedata;
6151
6152         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
6153                 return;
6154
6155         switch (kind) {
6156                 case RESET_KIND_INIT:
6157                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
6158                                         APE_HOST_SEG_SIG_MAGIC);
6159                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
6160                                         APE_HOST_SEG_LEN_MAGIC);
6161                         apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
6162                         tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
6163                         tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
6164                                         APE_HOST_DRIVER_ID_MAGIC);
6165                         tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
6166                                         APE_HOST_BEHAV_NO_PHYLOCK);
6167
6168                         event = APE_EVENT_STATUS_STATE_START;
6169                         break;
6170                 case RESET_KIND_SHUTDOWN:
6171                         /* With the interface we are currently using,
6172                          * APE does not track driver state.  Wiping
6173                          * out the HOST SEGMENT SIGNATURE forces
6174                          * the APE to assume OS absent status.
6175                          */
6176                         tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG, 0x0);
6177
6178                         event = APE_EVENT_STATUS_STATE_UNLOAD;
6179                         break;
6180                 case RESET_KIND_SUSPEND:
6181                         event = APE_EVENT_STATUS_STATE_SUSPEND;
6182                         break;
6183                 default:
6184                         return;
6185         }
6186
6187         event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
6188
6189         tg3_ape_send_event(tp, event);
6190 }
6191
6192 /* tp->lock is held. */
6193 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
6194 {
6195         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
6196                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
6197
6198         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
6199                 switch (kind) {
6200                 case RESET_KIND_INIT:
6201                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6202                                       DRV_STATE_START);
6203                         break;
6204
6205                 case RESET_KIND_SHUTDOWN:
6206                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6207                                       DRV_STATE_UNLOAD);
6208                         break;
6209
6210                 case RESET_KIND_SUSPEND:
6211                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6212                                       DRV_STATE_SUSPEND);
6213                         break;
6214
6215                 default:
6216                         break;
6217                 }
6218         }
6219
6220         if (kind == RESET_KIND_INIT ||
6221             kind == RESET_KIND_SUSPEND)
6222                 tg3_ape_driver_state_change(tp, kind);
6223 }
6224
6225 /* tp->lock is held. */
6226 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
6227 {
6228         if (tp->tg3_flags2 & TG3_FLG2_ASF_NEW_HANDSHAKE) {
6229                 switch (kind) {
6230                 case RESET_KIND_INIT:
6231                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6232                                       DRV_STATE_START_DONE);
6233                         break;
6234
6235                 case RESET_KIND_SHUTDOWN:
6236                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6237                                       DRV_STATE_UNLOAD_DONE);
6238                         break;
6239
6240                 default:
6241                         break;
6242                 }
6243         }
6244
6245         if (kind == RESET_KIND_SHUTDOWN)
6246                 tg3_ape_driver_state_change(tp, kind);
6247 }
6248
6249 /* tp->lock is held. */
6250 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
6251 {
6252         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
6253                 switch (kind) {
6254                 case RESET_KIND_INIT:
6255                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6256                                       DRV_STATE_START);
6257                         break;
6258
6259                 case RESET_KIND_SHUTDOWN:
6260                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6261                                       DRV_STATE_UNLOAD);
6262                         break;
6263
6264                 case RESET_KIND_SUSPEND:
6265                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6266                                       DRV_STATE_SUSPEND);
6267                         break;
6268
6269                 default:
6270                         break;
6271                 }
6272         }
6273 }
6274
6275 static int tg3_poll_fw(struct tg3 *tp)
6276 {
6277         int i;
6278         u32 val;
6279
6280         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
6281                 /* Wait up to 20ms for init done. */
6282                 for (i = 0; i < 200; i++) {
6283                         if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
6284                                 return 0;
6285                         udelay(100);
6286                 }
6287                 return -ENODEV;
6288         }
6289
6290         /* Wait for firmware initialization to complete. */
6291         for (i = 0; i < 100000; i++) {
6292                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
6293                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
6294                         break;
6295                 udelay(10);
6296         }
6297
6298         /* Chip might not be fitted with firmware.  Some Sun onboard
6299          * parts are configured like that.  So don't signal the timeout
6300          * of the above loop as an error, but do report the lack of
6301          * running firmware once.
6302          */
6303         if (i >= 100000 &&
6304             !(tp->tg3_flags2 & TG3_FLG2_NO_FWARE_REPORTED)) {
6305                 tp->tg3_flags2 |= TG3_FLG2_NO_FWARE_REPORTED;
6306
6307                 printk(KERN_INFO PFX "%s: No firmware running.\n",
6308                        tp->dev->name);
6309         }
6310
6311         return 0;
6312 }
6313
6314 /* Save PCI command register before chip reset */
6315 static void tg3_save_pci_state(struct tg3 *tp)
6316 {
6317         pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
6318 }
6319
6320 /* Restore PCI state after chip reset */
6321 static void tg3_restore_pci_state(struct tg3 *tp)
6322 {
6323         u32 val;
6324
6325         /* Re-enable indirect register accesses. */
6326         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
6327                                tp->misc_host_ctrl);
6328
6329         /* Set MAX PCI retry to zero. */
6330         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
6331         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
6332             (tp->tg3_flags & TG3_FLAG_PCIX_MODE))
6333                 val |= PCISTATE_RETRY_SAME_DMA;
6334         /* Allow reads and writes to the APE register and memory space. */
6335         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
6336                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
6337                        PCISTATE_ALLOW_APE_SHMEM_WR;
6338         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
6339
6340         pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
6341
6342         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785) {
6343                 if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
6344                         pcie_set_readrq(tp->pdev, 4096);
6345                 else {
6346                         pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
6347                                               tp->pci_cacheline_sz);
6348                         pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
6349                                               tp->pci_lat_timer);
6350                 }
6351         }
6352
6353         /* Make sure PCI-X relaxed ordering bit is clear. */
6354         if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
6355                 u16 pcix_cmd;
6356
6357                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
6358                                      &pcix_cmd);
6359                 pcix_cmd &= ~PCI_X_CMD_ERO;
6360                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
6361                                       pcix_cmd);
6362         }
6363
6364         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) {
6365
6366                 /* Chip reset on 5780 will reset MSI enable bit,
6367                  * so need to restore it.
6368                  */
6369                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
6370                         u16 ctrl;
6371
6372                         pci_read_config_word(tp->pdev,
6373                                              tp->msi_cap + PCI_MSI_FLAGS,
6374                                              &ctrl);
6375                         pci_write_config_word(tp->pdev,
6376                                               tp->msi_cap + PCI_MSI_FLAGS,
6377                                               ctrl | PCI_MSI_FLAGS_ENABLE);
6378                         val = tr32(MSGINT_MODE);
6379                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
6380                 }
6381         }
6382 }
6383
6384 static void tg3_stop_fw(struct tg3 *);
6385
6386 /* tp->lock is held. */
6387 static int tg3_chip_reset(struct tg3 *tp)
6388 {
6389         u32 val;
6390         void (*write_op)(struct tg3 *, u32, u32);
6391         int i, err;
6392
6393         tg3_nvram_lock(tp);
6394
6395         tg3_mdio_stop(tp);
6396
6397         tg3_ape_lock(tp, TG3_APE_LOCK_GRC);
6398
6399         /* No matching tg3_nvram_unlock() after this because
6400          * chip reset below will undo the nvram lock.
6401          */
6402         tp->nvram_lock_cnt = 0;
6403
6404         /* GRC_MISC_CFG core clock reset will clear the memory
6405          * enable bit in PCI register 4 and the MSI enable bit
6406          * on some chips, so we save relevant registers here.
6407          */
6408         tg3_save_pci_state(tp);
6409
6410         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
6411             (tp->tg3_flags3 & TG3_FLG3_5755_PLUS))
6412                 tw32(GRC_FASTBOOT_PC, 0);
6413
6414         /*
6415          * We must avoid the readl() that normally takes place.
6416          * It locks machines, causes machine checks, and other
6417          * fun things.  So, temporarily disable the 5701
6418          * hardware workaround, while we do the reset.
6419          */
6420         write_op = tp->write32;
6421         if (write_op == tg3_write_flush_reg32)
6422                 tp->write32 = tg3_write32;
6423
6424         /* Prevent the irq handler from reading or writing PCI registers
6425          * during chip reset when the memory enable bit in the PCI command
6426          * register may be cleared.  The chip does not generate interrupt
6427          * at this time, but the irq handler may still be called due to irq
6428          * sharing or irqpoll.
6429          */
6430         tp->tg3_flags |= TG3_FLAG_CHIP_RESETTING;
6431         for (i = 0; i < tp->irq_cnt; i++) {
6432                 struct tg3_napi *tnapi = &tp->napi[i];
6433                 if (tnapi->hw_status) {
6434                         tnapi->hw_status->status = 0;
6435                         tnapi->hw_status->status_tag = 0;
6436                 }
6437                 tnapi->last_tag = 0;
6438                 tnapi->last_irq_tag = 0;
6439         }
6440         smp_mb();
6441
6442         for (i = 0; i < tp->irq_cnt; i++)
6443                 synchronize_irq(tp->napi[i].irq_vec);
6444
6445         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780) {
6446                 val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
6447                 tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
6448         }
6449
6450         /* do the reset */
6451         val = GRC_MISC_CFG_CORECLK_RESET;
6452
6453         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
6454                 if (tr32(0x7e2c) == 0x60) {
6455                         tw32(0x7e2c, 0x20);
6456                 }
6457                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
6458                         tw32(GRC_MISC_CFG, (1 << 29));
6459                         val |= (1 << 29);
6460                 }
6461         }
6462
6463         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
6464                 tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
6465                 tw32(GRC_VCPU_EXT_CTRL,
6466                      tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
6467         }
6468
6469         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
6470                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
6471         tw32(GRC_MISC_CFG, val);
6472
6473         /* restore 5701 hardware bug workaround write method */
6474         tp->write32 = write_op;
6475
6476         /* Unfortunately, we have to delay before the PCI read back.
6477          * Some 575X chips even will not respond to a PCI cfg access
6478          * when the reset command is given to the chip.
6479          *
6480          * How do these hardware designers expect things to work
6481          * properly if the PCI write is posted for a long period
6482          * of time?  It is always necessary to have some method by
6483          * which a register read back can occur to push the write
6484          * out which does the reset.
6485          *
6486          * For most tg3 variants the trick below was working.
6487          * Ho hum...
6488          */
6489         udelay(120);
6490
6491         /* Flush PCI posted writes.  The normal MMIO registers
6492          * are inaccessible at this time so this is the only
6493          * way to make this reliably (actually, this is no longer
6494          * the case, see above).  I tried to use indirect
6495          * register read/write but this upset some 5701 variants.
6496          */
6497         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
6498
6499         udelay(120);
6500
6501         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) && tp->pcie_cap) {
6502                 u16 val16;
6503
6504                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
6505                         int i;
6506                         u32 cfg_val;
6507
6508                         /* Wait for link training to complete.  */
6509                         for (i = 0; i < 5000; i++)
6510                                 udelay(100);
6511
6512                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
6513                         pci_write_config_dword(tp->pdev, 0xc4,
6514                                                cfg_val | (1 << 15));
6515                 }
6516
6517                 /* Clear the "no snoop" and "relaxed ordering" bits. */
6518                 pci_read_config_word(tp->pdev,
6519                                      tp->pcie_cap + PCI_EXP_DEVCTL,
6520                                      &val16);
6521                 val16 &= ~(PCI_EXP_DEVCTL_RELAX_EN |
6522                            PCI_EXP_DEVCTL_NOSNOOP_EN);
6523                 /*
6524                  * Older PCIe devices only support the 128 byte
6525                  * MPS setting.  Enforce the restriction.
6526                  */
6527                 if (!(tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) ||
6528                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784))
6529                         val16 &= ~PCI_EXP_DEVCTL_PAYLOAD;
6530                 pci_write_config_word(tp->pdev,
6531                                       tp->pcie_cap + PCI_EXP_DEVCTL,
6532                                       val16);
6533
6534                 pcie_set_readrq(tp->pdev, 4096);
6535
6536                 /* Clear error status */
6537                 pci_write_config_word(tp->pdev,
6538                                       tp->pcie_cap + PCI_EXP_DEVSTA,
6539                                       PCI_EXP_DEVSTA_CED |
6540                                       PCI_EXP_DEVSTA_NFED |
6541                                       PCI_EXP_DEVSTA_FED |
6542                                       PCI_EXP_DEVSTA_URD);
6543         }
6544
6545         tg3_restore_pci_state(tp);
6546
6547         tp->tg3_flags &= ~TG3_FLAG_CHIP_RESETTING;
6548
6549         val = 0;
6550         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
6551                 val = tr32(MEMARB_MODE);
6552         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
6553
6554         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
6555                 tg3_stop_fw(tp);
6556                 tw32(0x5000, 0x400);
6557         }
6558
6559         tw32(GRC_MODE, tp->grc_mode);
6560
6561         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
6562                 val = tr32(0xc4);
6563
6564                 tw32(0xc4, val | (1 << 15));
6565         }
6566
6567         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
6568             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6569                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
6570                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
6571                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
6572                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
6573         }
6574
6575         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
6576                 tp->mac_mode = MAC_MODE_PORT_MODE_TBI;
6577                 tw32_f(MAC_MODE, tp->mac_mode);
6578         } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
6579                 tp->mac_mode = MAC_MODE_PORT_MODE_GMII;
6580                 tw32_f(MAC_MODE, tp->mac_mode);
6581         } else if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
6582                 tp->mac_mode &= (MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN);
6583                 if (tp->mac_mode & MAC_MODE_APE_TX_EN)
6584                         tp->mac_mode |= MAC_MODE_TDE_ENABLE;
6585                 tw32_f(MAC_MODE, tp->mac_mode);
6586         } else
6587                 tw32_f(MAC_MODE, 0);
6588         udelay(40);
6589
6590         tg3_ape_unlock(tp, TG3_APE_LOCK_GRC);
6591
6592         err = tg3_poll_fw(tp);
6593         if (err)
6594                 return err;
6595
6596         tg3_mdio_start(tp);
6597
6598         if ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
6599             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
6600             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785 &&
6601             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717) {
6602                 val = tr32(0x7c00);
6603
6604                 tw32(0x7c00, val | (1 << 25));
6605         }
6606
6607         /* Reprobe ASF enable state.  */
6608         tp->tg3_flags &= ~TG3_FLAG_ENABLE_ASF;
6609         tp->tg3_flags2 &= ~TG3_FLG2_ASF_NEW_HANDSHAKE;
6610         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
6611         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
6612                 u32 nic_cfg;
6613
6614                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
6615                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
6616                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
6617                         tp->last_event_jiffies = jiffies;
6618                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
6619                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
6620                 }
6621         }
6622
6623         return 0;
6624 }
6625
6626 /* tp->lock is held. */
6627 static void tg3_stop_fw(struct tg3 *tp)
6628 {
6629         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
6630            !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
6631                 /* Wait for RX cpu to ACK the previous event. */
6632                 tg3_wait_for_event_ack(tp);
6633
6634                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
6635
6636                 tg3_generate_fw_event(tp);
6637
6638                 /* Wait for RX cpu to ACK this event. */
6639                 tg3_wait_for_event_ack(tp);
6640         }
6641 }
6642
6643 /* tp->lock is held. */
6644 static int tg3_halt(struct tg3 *tp, int kind, int silent)
6645 {
6646         int err;
6647
6648         tg3_stop_fw(tp);
6649
6650         tg3_write_sig_pre_reset(tp, kind);
6651
6652         tg3_abort_hw(tp, silent);
6653         err = tg3_chip_reset(tp);
6654
6655         __tg3_set_mac_addr(tp, 0);
6656
6657         tg3_write_sig_legacy(tp, kind);
6658         tg3_write_sig_post_reset(tp, kind);
6659
6660         if (err)
6661                 return err;
6662
6663         return 0;
6664 }
6665
6666 #define RX_CPU_SCRATCH_BASE     0x30000
6667 #define RX_CPU_SCRATCH_SIZE     0x04000
6668 #define TX_CPU_SCRATCH_BASE     0x34000
6669 #define TX_CPU_SCRATCH_SIZE     0x04000
6670
6671 /* tp->lock is held. */
6672 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
6673 {
6674         int i;
6675
6676         BUG_ON(offset == TX_CPU_BASE &&
6677             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS));
6678
6679         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
6680                 u32 val = tr32(GRC_VCPU_EXT_CTRL);
6681
6682                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
6683                 return 0;
6684         }
6685         if (offset == RX_CPU_BASE) {
6686                 for (i = 0; i < 10000; i++) {
6687                         tw32(offset + CPU_STATE, 0xffffffff);
6688                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
6689                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
6690                                 break;
6691                 }
6692
6693                 tw32(offset + CPU_STATE, 0xffffffff);
6694                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
6695                 udelay(10);
6696         } else {
6697                 for (i = 0; i < 10000; i++) {
6698                         tw32(offset + CPU_STATE, 0xffffffff);
6699                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
6700                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
6701                                 break;
6702                 }
6703         }
6704
6705         if (i >= 10000) {
6706                 printk(KERN_ERR PFX "tg3_reset_cpu timed out for %s, "
6707                        "and %s CPU\n",
6708                        tp->dev->name,
6709                        (offset == RX_CPU_BASE ? "RX" : "TX"));
6710                 return -ENODEV;
6711         }
6712
6713         /* Clear firmware's nvram arbitration. */
6714         if (tp->tg3_flags & TG3_FLAG_NVRAM)
6715                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
6716         return 0;
6717 }
6718
6719 struct fw_info {
6720         unsigned int fw_base;
6721         unsigned int fw_len;
6722         const __be32 *fw_data;
6723 };
6724
6725 /* tp->lock is held. */
6726 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
6727                                  int cpu_scratch_size, struct fw_info *info)
6728 {
6729         int err, lock_err, i;
6730         void (*write_op)(struct tg3 *, u32, u32);
6731
6732         if (cpu_base == TX_CPU_BASE &&
6733             (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6734                 printk(KERN_ERR PFX "tg3_load_firmware_cpu: Trying to load "
6735                        "TX cpu firmware on %s which is 5705.\n",
6736                        tp->dev->name);
6737                 return -EINVAL;
6738         }
6739
6740         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
6741                 write_op = tg3_write_mem;
6742         else
6743                 write_op = tg3_write_indirect_reg32;
6744
6745         /* It is possible that bootcode is still loading at this point.
6746          * Get the nvram lock first before halting the cpu.
6747          */
6748         lock_err = tg3_nvram_lock(tp);
6749         err = tg3_halt_cpu(tp, cpu_base);
6750         if (!lock_err)
6751                 tg3_nvram_unlock(tp);
6752         if (err)
6753                 goto out;
6754
6755         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
6756                 write_op(tp, cpu_scratch_base + i, 0);
6757         tw32(cpu_base + CPU_STATE, 0xffffffff);
6758         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
6759         for (i = 0; i < (info->fw_len / sizeof(u32)); i++)
6760                 write_op(tp, (cpu_scratch_base +
6761                               (info->fw_base & 0xffff) +
6762                               (i * sizeof(u32))),
6763                               be32_to_cpu(info->fw_data[i]));
6764
6765         err = 0;
6766
6767 out:
6768         return err;
6769 }
6770
6771 /* tp->lock is held. */
6772 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
6773 {
6774         struct fw_info info;
6775         const __be32 *fw_data;
6776         int err, i;
6777
6778         fw_data = (void *)tp->fw->data;
6779
6780         /* Firmware blob starts with version numbers, followed by
6781            start address and length. We are setting complete length.
6782            length = end_address_of_bss - start_address_of_text.
6783            Remainder is the blob to be loaded contiguously
6784            from start address. */
6785
6786         info.fw_base = be32_to_cpu(fw_data[1]);
6787         info.fw_len = tp->fw->size - 12;
6788         info.fw_data = &fw_data[3];
6789
6790         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
6791                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
6792                                     &info);
6793         if (err)
6794                 return err;
6795
6796         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
6797                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
6798                                     &info);
6799         if (err)
6800                 return err;
6801
6802         /* Now startup only the RX cpu. */
6803         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
6804         tw32_f(RX_CPU_BASE + CPU_PC, info.fw_base);
6805
6806         for (i = 0; i < 5; i++) {
6807                 if (tr32(RX_CPU_BASE + CPU_PC) == info.fw_base)
6808                         break;
6809                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
6810                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
6811                 tw32_f(RX_CPU_BASE + CPU_PC, info.fw_base);
6812                 udelay(1000);
6813         }
6814         if (i >= 5) {
6815                 printk(KERN_ERR PFX "tg3_load_firmware fails for %s "
6816                        "to set RX CPU PC, is %08x should be %08x\n",
6817                        tp->dev->name, tr32(RX_CPU_BASE + CPU_PC),
6818                        info.fw_base);
6819                 return -ENODEV;
6820         }
6821         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
6822         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
6823
6824         return 0;
6825 }
6826
6827 /* 5705 needs a special version of the TSO firmware.  */
6828
6829 /* tp->lock is held. */
6830 static int tg3_load_tso_firmware(struct tg3 *tp)
6831 {
6832         struct fw_info info;
6833         const __be32 *fw_data;
6834         unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
6835         int err, i;
6836
6837         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
6838                 return 0;
6839
6840         fw_data = (void *)tp->fw->data;
6841
6842         /* Firmware blob starts with version numbers, followed by
6843            start address and length. We are setting complete length.
6844            length = end_address_of_bss - start_address_of_text.
6845            Remainder is the blob to be loaded contiguously
6846            from start address. */
6847
6848         info.fw_base = be32_to_cpu(fw_data[1]);
6849         cpu_scratch_size = tp->fw_len;
6850         info.fw_len = tp->fw->size - 12;
6851         info.fw_data = &fw_data[3];
6852
6853         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
6854                 cpu_base = RX_CPU_BASE;
6855                 cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
6856         } else {
6857                 cpu_base = TX_CPU_BASE;
6858                 cpu_scratch_base = TX_CPU_SCRATCH_BASE;
6859                 cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
6860         }
6861
6862         err = tg3_load_firmware_cpu(tp, cpu_base,
6863                                     cpu_scratch_base, cpu_scratch_size,
6864                                     &info);
6865         if (err)
6866                 return err;
6867
6868         /* Now startup the cpu. */
6869         tw32(cpu_base + CPU_STATE, 0xffffffff);
6870         tw32_f(cpu_base + CPU_PC, info.fw_base);
6871
6872         for (i = 0; i < 5; i++) {
6873                 if (tr32(cpu_base + CPU_PC) == info.fw_base)
6874                         break;
6875                 tw32(cpu_base + CPU_STATE, 0xffffffff);
6876                 tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
6877                 tw32_f(cpu_base + CPU_PC, info.fw_base);
6878                 udelay(1000);
6879         }
6880         if (i >= 5) {
6881                 printk(KERN_ERR PFX "tg3_load_tso_firmware fails for %s "
6882                        "to set CPU PC, is %08x should be %08x\n",
6883                        tp->dev->name, tr32(cpu_base + CPU_PC),
6884                        info.fw_base);
6885                 return -ENODEV;
6886         }
6887         tw32(cpu_base + CPU_STATE, 0xffffffff);
6888         tw32_f(cpu_base + CPU_MODE,  0x00000000);
6889         return 0;
6890 }
6891
6892
6893 static int tg3_set_mac_addr(struct net_device *dev, void *p)
6894 {
6895         struct tg3 *tp = netdev_priv(dev);
6896         struct sockaddr *addr = p;
6897         int err = 0, skip_mac_1 = 0;
6898
6899         if (!is_valid_ether_addr(addr->sa_data))
6900                 return -EINVAL;
6901
6902         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
6903
6904         if (!netif_running(dev))
6905                 return 0;
6906
6907         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) {
6908                 u32 addr0_high, addr0_low, addr1_high, addr1_low;
6909
6910                 addr0_high = tr32(MAC_ADDR_0_HIGH);
6911                 addr0_low = tr32(MAC_ADDR_0_LOW);
6912                 addr1_high = tr32(MAC_ADDR_1_HIGH);
6913                 addr1_low = tr32(MAC_ADDR_1_LOW);
6914
6915                 /* Skip MAC addr 1 if ASF is using it. */
6916                 if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
6917                     !(addr1_high == 0 && addr1_low == 0))
6918                         skip_mac_1 = 1;
6919         }
6920         spin_lock_bh(&tp->lock);
6921         __tg3_set_mac_addr(tp, skip_mac_1);
6922         spin_unlock_bh(&tp->lock);
6923
6924         return err;
6925 }
6926
6927 /* tp->lock is held. */
6928 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
6929                            dma_addr_t mapping, u32 maxlen_flags,
6930                            u32 nic_addr)
6931 {
6932         tg3_write_mem(tp,
6933                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
6934                       ((u64) mapping >> 32));
6935         tg3_write_mem(tp,
6936                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
6937                       ((u64) mapping & 0xffffffff));
6938         tg3_write_mem(tp,
6939                       (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
6940                        maxlen_flags);
6941
6942         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
6943                 tg3_write_mem(tp,
6944                               (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
6945                               nic_addr);
6946 }
6947
6948 static void __tg3_set_rx_mode(struct net_device *);
6949 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
6950 {
6951         int i;
6952
6953         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSIX)) {
6954                 tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
6955                 tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
6956                 tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
6957
6958                 tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
6959                 tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
6960                 tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
6961         } else {
6962                 tw32(HOSTCC_TXCOL_TICKS, 0);
6963                 tw32(HOSTCC_TXMAX_FRAMES, 0);
6964                 tw32(HOSTCC_TXCOAL_MAXF_INT, 0);
6965
6966                 tw32(HOSTCC_RXCOL_TICKS, 0);
6967                 tw32(HOSTCC_RXMAX_FRAMES, 0);
6968                 tw32(HOSTCC_RXCOAL_MAXF_INT, 0);
6969         }
6970
6971         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
6972                 u32 val = ec->stats_block_coalesce_usecs;
6973
6974                 tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
6975                 tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
6976
6977                 if (!netif_carrier_ok(tp->dev))
6978                         val = 0;
6979
6980                 tw32(HOSTCC_STAT_COAL_TICKS, val);
6981         }
6982
6983         for (i = 0; i < tp->irq_cnt - 1; i++) {
6984                 u32 reg;
6985
6986                 reg = HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18;
6987                 tw32(reg, ec->rx_coalesce_usecs);
6988                 reg = HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18;
6989                 tw32(reg, ec->tx_coalesce_usecs);
6990                 reg = HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18;
6991                 tw32(reg, ec->rx_max_coalesced_frames);
6992                 reg = HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18;
6993                 tw32(reg, ec->tx_max_coalesced_frames);
6994                 reg = HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18;
6995                 tw32(reg, ec->rx_max_coalesced_frames_irq);
6996                 reg = HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18;
6997                 tw32(reg, ec->tx_max_coalesced_frames_irq);
6998         }
6999
7000         for (; i < tp->irq_max - 1; i++) {
7001                 tw32(HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18, 0);
7002                 tw32(HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18, 0);
7003                 tw32(HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18, 0);
7004                 tw32(HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18, 0);
7005                 tw32(HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
7006                 tw32(HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
7007         }
7008 }
7009
7010 /* tp->lock is held. */
7011 static void tg3_rings_reset(struct tg3 *tp)
7012 {
7013         int i;
7014         u32 stblk, txrcb, rxrcb, limit;
7015         struct tg3_napi *tnapi = &tp->napi[0];
7016
7017         /* Disable all transmit rings but the first. */
7018         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
7019                 limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 16;
7020         else
7021                 limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
7022
7023         for (txrcb = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
7024              txrcb < limit; txrcb += TG3_BDINFO_SIZE)
7025                 tg3_write_mem(tp, txrcb + TG3_BDINFO_MAXLEN_FLAGS,
7026                               BDINFO_FLAGS_DISABLED);
7027
7028
7029         /* Disable all receive return rings but the first. */
7030         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
7031                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 17;
7032         else if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
7033                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 16;
7034         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
7035                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 4;
7036         else
7037                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
7038
7039         for (rxrcb = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
7040              rxrcb < limit; rxrcb += TG3_BDINFO_SIZE)
7041                 tg3_write_mem(tp, rxrcb + TG3_BDINFO_MAXLEN_FLAGS,
7042                               BDINFO_FLAGS_DISABLED);
7043
7044         /* Disable interrupts */
7045         tw32_mailbox_f(tp->napi[0].int_mbox, 1);
7046
7047         /* Zero mailbox registers. */
7048         if (tp->tg3_flags & TG3_FLAG_SUPPORT_MSIX) {
7049                 for (i = 1; i < TG3_IRQ_MAX_VECS; i++) {
7050                         tp->napi[i].tx_prod = 0;
7051                         tp->napi[i].tx_cons = 0;
7052                         tw32_mailbox(tp->napi[i].prodmbox, 0);
7053                         tw32_rx_mbox(tp->napi[i].consmbox, 0);
7054                         tw32_mailbox_f(tp->napi[i].int_mbox, 1);
7055                 }
7056         } else {
7057                 tp->napi[0].tx_prod = 0;
7058                 tp->napi[0].tx_cons = 0;
7059                 tw32_mailbox(tp->napi[0].prodmbox, 0);
7060                 tw32_rx_mbox(tp->napi[0].consmbox, 0);
7061         }
7062
7063         /* Make sure the NIC-based send BD rings are disabled. */
7064         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
7065                 u32 mbox = MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW;
7066                 for (i = 0; i < 16; i++)
7067                         tw32_tx_mbox(mbox + i * 8, 0);
7068         }
7069
7070         txrcb = NIC_SRAM_SEND_RCB;
7071         rxrcb = NIC_SRAM_RCV_RET_RCB;
7072
7073         /* Clear status block in ram. */
7074         memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
7075
7076         /* Set status block DMA address */
7077         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
7078              ((u64) tnapi->status_mapping >> 32));
7079         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
7080              ((u64) tnapi->status_mapping & 0xffffffff));
7081
7082         if (tnapi->tx_ring) {
7083                 tg3_set_bdinfo(tp, txrcb, tnapi->tx_desc_mapping,
7084                                (TG3_TX_RING_SIZE <<
7085                                 BDINFO_FLAGS_MAXLEN_SHIFT),
7086                                NIC_SRAM_TX_BUFFER_DESC);
7087                 txrcb += TG3_BDINFO_SIZE;
7088         }
7089
7090         if (tnapi->rx_rcb) {
7091                 tg3_set_bdinfo(tp, rxrcb, tnapi->rx_rcb_mapping,
7092                                (TG3_RX_RCB_RING_SIZE(tp) <<
7093                                 BDINFO_FLAGS_MAXLEN_SHIFT), 0);
7094                 rxrcb += TG3_BDINFO_SIZE;
7095         }
7096
7097         stblk = HOSTCC_STATBLCK_RING1;
7098
7099         for (i = 1, tnapi++; i < tp->irq_cnt; i++, tnapi++) {
7100                 u64 mapping = (u64)tnapi->status_mapping;
7101                 tw32(stblk + TG3_64BIT_REG_HIGH, mapping >> 32);
7102                 tw32(stblk + TG3_64BIT_REG_LOW, mapping & 0xffffffff);
7103
7104                 /* Clear status block in ram. */
7105                 memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
7106
7107                 tg3_set_bdinfo(tp, txrcb, tnapi->tx_desc_mapping,
7108                                (TG3_TX_RING_SIZE <<
7109                                 BDINFO_FLAGS_MAXLEN_SHIFT),
7110                                NIC_SRAM_TX_BUFFER_DESC);
7111
7112                 tg3_set_bdinfo(tp, rxrcb, tnapi->rx_rcb_mapping,
7113                                (TG3_RX_RCB_RING_SIZE(tp) <<
7114                                 BDINFO_FLAGS_MAXLEN_SHIFT), 0);
7115
7116                 stblk += 8;
7117                 txrcb += TG3_BDINFO_SIZE;
7118                 rxrcb += TG3_BDINFO_SIZE;
7119         }
7120 }
7121
7122 /* tp->lock is held. */
7123 static int tg3_reset_hw(struct tg3 *tp, int reset_phy)
7124 {
7125         u32 val, rdmac_mode;
7126         int i, err, limit;
7127         struct tg3_rx_prodring_set *tpr = &tp->prodring[0];
7128
7129         tg3_disable_ints(tp);
7130
7131         tg3_stop_fw(tp);
7132
7133         tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
7134
7135         if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
7136                 tg3_abort_hw(tp, 1);
7137         }
7138
7139         if (reset_phy &&
7140             !(tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB))
7141                 tg3_phy_reset(tp);
7142
7143         err = tg3_chip_reset(tp);
7144         if (err)
7145                 return err;
7146
7147         tg3_write_sig_legacy(tp, RESET_KIND_INIT);
7148
7149         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX) {
7150                 val = tr32(TG3_CPMU_CTRL);
7151                 val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
7152                 tw32(TG3_CPMU_CTRL, val);
7153
7154                 val = tr32(TG3_CPMU_LSPD_10MB_CLK);
7155                 val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
7156                 val |= CPMU_LSPD_10MB_MACCLK_6_25;
7157                 tw32(TG3_CPMU_LSPD_10MB_CLK, val);
7158
7159                 val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
7160                 val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
7161                 val |= CPMU_LNK_AWARE_MACCLK_6_25;
7162                 tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
7163
7164                 val = tr32(TG3_CPMU_HST_ACC);
7165                 val &= ~CPMU_HST_ACC_MACCLK_MASK;
7166                 val |= CPMU_HST_ACC_MACCLK_6_25;
7167                 tw32(TG3_CPMU_HST_ACC, val);
7168         }
7169
7170         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780) {
7171                 val = tr32(PCIE_PWR_MGMT_THRESH) & ~PCIE_PWR_MGMT_L1_THRESH_MSK;
7172                 val |= PCIE_PWR_MGMT_EXT_ASPM_TMR_EN |
7173                        PCIE_PWR_MGMT_L1_THRESH_4MS;
7174                 tw32(PCIE_PWR_MGMT_THRESH, val);
7175
7176                 val = tr32(TG3_PCIE_EIDLE_DELAY) & ~TG3_PCIE_EIDLE_DELAY_MASK;
7177                 tw32(TG3_PCIE_EIDLE_DELAY, val | TG3_PCIE_EIDLE_DELAY_13_CLKS);
7178
7179                 tw32(TG3_CORR_ERR_STAT, TG3_CORR_ERR_STAT_CLEAR);
7180         }
7181
7182         if (tp->tg3_flags3 & TG3_FLG3_TOGGLE_10_100_L1PLLPD) {
7183                 val = tr32(TG3_PCIE_LNKCTL);
7184                 if (tp->tg3_flags3 & TG3_FLG3_CLKREQ_BUG)
7185                         val |= TG3_PCIE_LNKCTL_L1_PLL_PD_DIS;
7186                 else
7187                         val &= ~TG3_PCIE_LNKCTL_L1_PLL_PD_DIS;
7188                 tw32(TG3_PCIE_LNKCTL, val);
7189         }
7190
7191         /* This works around an issue with Athlon chipsets on
7192          * B3 tigon3 silicon.  This bit has no effect on any
7193          * other revision.  But do not set this on PCI Express
7194          * chips and don't even touch the clocks if the CPMU is present.
7195          */
7196         if (!(tp->tg3_flags & TG3_FLAG_CPMU_PRESENT)) {
7197                 if (!(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
7198                         tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
7199                 tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
7200         }
7201
7202         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
7203             (tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
7204                 val = tr32(TG3PCI_PCISTATE);
7205                 val |= PCISTATE_RETRY_SAME_DMA;
7206                 tw32(TG3PCI_PCISTATE, val);
7207         }
7208
7209         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
7210                 /* Allow reads and writes to the
7211                  * APE register and memory space.
7212                  */
7213                 val = tr32(TG3PCI_PCISTATE);
7214                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
7215                        PCISTATE_ALLOW_APE_SHMEM_WR;
7216                 tw32(TG3PCI_PCISTATE, val);
7217         }
7218
7219         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_BX) {
7220                 /* Enable some hw fixes.  */
7221                 val = tr32(TG3PCI_MSI_DATA);
7222                 val |= (1 << 26) | (1 << 28) | (1 << 29);
7223                 tw32(TG3PCI_MSI_DATA, val);
7224         }
7225
7226         /* Descriptor ring init may make accesses to the
7227          * NIC SRAM area to setup the TX descriptors, so we
7228          * can only do this after the hardware has been
7229          * successfully reset.
7230          */
7231         err = tg3_init_rings(tp);
7232         if (err)
7233                 return err;
7234
7235         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
7236             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761 &&
7237             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717) {
7238                 /* This value is determined during the probe time DMA
7239                  * engine test, tg3_test_dma.
7240                  */
7241                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
7242         }
7243
7244         tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
7245                           GRC_MODE_4X_NIC_SEND_RINGS |
7246                           GRC_MODE_NO_TX_PHDR_CSUM |
7247                           GRC_MODE_NO_RX_PHDR_CSUM);
7248         tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
7249
7250         /* Pseudo-header checksum is done by hardware logic and not
7251          * the offload processers, so make the chip do the pseudo-
7252          * header checksums on receive.  For transmit it is more
7253          * convenient to do the pseudo-header checksum in software
7254          * as Linux does that on transmit for us in all cases.
7255          */
7256         tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
7257
7258         tw32(GRC_MODE,
7259              tp->grc_mode |
7260              (GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP));
7261
7262         /* Setup the timer prescalar register.  Clock is always 66Mhz. */
7263         val = tr32(GRC_MISC_CFG);
7264         val &= ~0xff;
7265         val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
7266         tw32(GRC_MISC_CFG, val);
7267
7268         /* Initialize MBUF/DESC pool. */
7269         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
7270                 /* Do nothing.  */
7271         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705) {
7272                 tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
7273                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
7274                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
7275                 else
7276                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
7277                 tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
7278                 tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
7279         }
7280         else if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
7281                 int fw_len;
7282
7283                 fw_len = tp->fw_len;
7284                 fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
7285                 tw32(BUFMGR_MB_POOL_ADDR,
7286                      NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
7287                 tw32(BUFMGR_MB_POOL_SIZE,
7288                      NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
7289         }
7290
7291         if (tp->dev->mtu <= ETH_DATA_LEN) {
7292                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
7293                      tp->bufmgr_config.mbuf_read_dma_low_water);
7294                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
7295                      tp->bufmgr_config.mbuf_mac_rx_low_water);
7296                 tw32(BUFMGR_MB_HIGH_WATER,
7297                      tp->bufmgr_config.mbuf_high_water);
7298         } else {
7299                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
7300                      tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
7301                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
7302                      tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
7303                 tw32(BUFMGR_MB_HIGH_WATER,
7304                      tp->bufmgr_config.mbuf_high_water_jumbo);
7305         }
7306         tw32(BUFMGR_DMA_LOW_WATER,
7307              tp->bufmgr_config.dma_low_water);
7308         tw32(BUFMGR_DMA_HIGH_WATER,
7309              tp->bufmgr_config.dma_high_water);
7310
7311         tw32(BUFMGR_MODE, BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE);
7312         for (i = 0; i < 2000; i++) {
7313                 if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
7314                         break;
7315                 udelay(10);
7316         }
7317         if (i >= 2000) {
7318                 printk(KERN_ERR PFX "tg3_reset_hw cannot enable BUFMGR for %s.\n",
7319                        tp->dev->name);
7320                 return -ENODEV;
7321         }
7322
7323         /* Setup replenish threshold. */
7324         val = tp->rx_pending / 8;
7325         if (val == 0)
7326                 val = 1;
7327         else if (val > tp->rx_std_max_post)
7328                 val = tp->rx_std_max_post;
7329         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
7330                 if (tp->pci_chip_rev_id == CHIPREV_ID_5906_A1)
7331                         tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
7332
7333                 if (val > (TG3_RX_INTERNAL_RING_SZ_5906 / 2))
7334                         val = TG3_RX_INTERNAL_RING_SZ_5906 / 2;
7335         }
7336
7337         tw32(RCVBDI_STD_THRESH, val);
7338
7339         /* Initialize TG3_BDINFO's at:
7340          *  RCVDBDI_STD_BD:     standard eth size rx ring
7341          *  RCVDBDI_JUMBO_BD:   jumbo frame rx ring
7342          *  RCVDBDI_MINI_BD:    small frame rx ring (??? does not work)
7343          *
7344          * like so:
7345          *  TG3_BDINFO_HOST_ADDR:       high/low parts of DMA address of ring
7346          *  TG3_BDINFO_MAXLEN_FLAGS:    (rx max buffer size << 16) |
7347          *                              ring attribute flags
7348          *  TG3_BDINFO_NIC_ADDR:        location of descriptors in nic SRAM
7349          *
7350          * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
7351          * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
7352          *
7353          * The size of each ring is fixed in the firmware, but the location is
7354          * configurable.
7355          */
7356         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
7357              ((u64) tpr->rx_std_mapping >> 32));
7358         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
7359              ((u64) tpr->rx_std_mapping & 0xffffffff));
7360         tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
7361              NIC_SRAM_RX_BUFFER_DESC);
7362
7363         /* Disable the mini ring */
7364         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
7365                 tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
7366                      BDINFO_FLAGS_DISABLED);
7367
7368         /* Program the jumbo buffer descriptor ring control
7369          * blocks on those devices that have them.
7370          */
7371         if ((tp->tg3_flags & TG3_FLAG_JUMBO_CAPABLE) &&
7372             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
7373                 /* Setup replenish threshold. */
7374                 tw32(RCVBDI_JUMBO_THRESH, tp->rx_jumbo_pending / 8);
7375
7376                 if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) {
7377                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
7378                              ((u64) tpr->rx_jmb_mapping >> 32));
7379                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
7380                              ((u64) tpr->rx_jmb_mapping & 0xffffffff));
7381                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
7382                              (RX_JUMBO_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT) |
7383                              BDINFO_FLAGS_USE_EXT_RECV);
7384                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
7385                              NIC_SRAM_RX_JUMBO_BUFFER_DESC);
7386                 } else {
7387                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
7388                              BDINFO_FLAGS_DISABLED);
7389                 }
7390
7391                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
7392                         val = (RX_STD_MAX_SIZE_5705 << BDINFO_FLAGS_MAXLEN_SHIFT) |
7393                               (RX_STD_MAX_SIZE << 2);
7394                 else
7395                         val = RX_STD_MAX_SIZE << BDINFO_FLAGS_MAXLEN_SHIFT;
7396         } else
7397                 val = RX_STD_MAX_SIZE_5705 << BDINFO_FLAGS_MAXLEN_SHIFT;
7398
7399         tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS, val);
7400
7401         tpr->rx_std_ptr = tp->rx_pending;
7402         tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
7403                      tpr->rx_std_ptr);
7404
7405         tpr->rx_jmb_ptr = (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE) ?
7406                           tp->rx_jumbo_pending : 0;
7407         tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
7408                      tpr->rx_jmb_ptr);
7409
7410         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
7411                 tw32(STD_REPLENISH_LWM, 32);
7412                 tw32(JMB_REPLENISH_LWM, 16);
7413         }
7414
7415         tg3_rings_reset(tp);
7416
7417         /* Initialize MAC address and backoff seed. */
7418         __tg3_set_mac_addr(tp, 0);
7419
7420         /* MTU + ethernet header + FCS + optional VLAN tag */
7421         tw32(MAC_RX_MTU_SIZE,
7422              tp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
7423
7424         /* The slot time is changed by tg3_setup_phy if we
7425          * run at gigabit with half duplex.
7426          */
7427         tw32(MAC_TX_LENGTHS,
7428              (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
7429              (6 << TX_LENGTHS_IPG_SHIFT) |
7430              (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
7431
7432         /* Receive rules. */
7433         tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
7434         tw32(RCVLPC_CONFIG, 0x0181);
7435
7436         /* Calculate RDMAC_MODE setting early, we need it to determine
7437          * the RCVLPC_STATE_ENABLE mask.
7438          */
7439         rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
7440                       RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
7441                       RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
7442                       RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
7443                       RDMAC_MODE_LNGREAD_ENAB);
7444
7445         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
7446             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
7447             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
7448                 rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
7449                               RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
7450                               RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
7451
7452         /* If statement applies to 5705 and 5750 PCI devices only */
7453         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
7454              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
7455             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)) {
7456                 if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE &&
7457                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
7458                         rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
7459                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
7460                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788)) {
7461                         rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
7462                 }
7463         }
7464
7465         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)
7466                 rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
7467
7468         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
7469                 rdmac_mode |= RDMAC_MODE_IPV4_LSO_EN;
7470
7471         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
7472             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
7473                 rdmac_mode |= RDMAC_MODE_IPV6_LSO_EN;
7474
7475         /* Receive/send statistics. */
7476         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
7477                 val = tr32(RCVLPC_STATS_ENABLE);
7478                 val &= ~RCVLPC_STATSENAB_DACK_FIX;
7479                 tw32(RCVLPC_STATS_ENABLE, val);
7480         } else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
7481                    (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
7482                 val = tr32(RCVLPC_STATS_ENABLE);
7483                 val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
7484                 tw32(RCVLPC_STATS_ENABLE, val);
7485         } else {
7486                 tw32(RCVLPC_STATS_ENABLE, 0xffffff);
7487         }
7488         tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
7489         tw32(SNDDATAI_STATSENAB, 0xffffff);
7490         tw32(SNDDATAI_STATSCTRL,
7491              (SNDDATAI_SCTRL_ENABLE |
7492               SNDDATAI_SCTRL_FASTUPD));
7493
7494         /* Setup host coalescing engine. */
7495         tw32(HOSTCC_MODE, 0);
7496         for (i = 0; i < 2000; i++) {
7497                 if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
7498                         break;
7499                 udelay(10);
7500         }
7501
7502         __tg3_set_coalesce(tp, &tp->coal);
7503
7504         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
7505                 /* Status/statistics block address.  See tg3_timer,
7506                  * the tg3_periodic_fetch_stats call there, and
7507                  * tg3_get_stats to see how this works for 5705/5750 chips.
7508                  */
7509                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
7510                      ((u64) tp->stats_mapping >> 32));
7511                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
7512                      ((u64) tp->stats_mapping & 0xffffffff));
7513                 tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
7514
7515                 tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
7516
7517                 /* Clear statistics and status block memory areas */
7518                 for (i = NIC_SRAM_STATS_BLK;
7519                      i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
7520                      i += sizeof(u32)) {
7521                         tg3_write_mem(tp, i, 0);
7522                         udelay(40);
7523                 }
7524         }
7525
7526         tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
7527
7528         tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
7529         tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
7530         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
7531                 tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
7532
7533         if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
7534                 tp->tg3_flags2 &= ~TG3_FLG2_PARALLEL_DETECT;
7535                 /* reset to prevent losing 1st rx packet intermittently */
7536                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
7537                 udelay(10);
7538         }
7539
7540         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
7541                 tp->mac_mode &= MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
7542         else
7543                 tp->mac_mode = 0;
7544         tp->mac_mode |= MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
7545                 MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE | MAC_MODE_FHDE_ENABLE;
7546         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
7547             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7548             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700)
7549                 tp->mac_mode |= MAC_MODE_LINK_POLARITY;
7550         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
7551         udelay(40);
7552
7553         /* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
7554          * If TG3_FLG2_IS_NIC is zero, we should read the
7555          * register to preserve the GPIO settings for LOMs. The GPIOs,
7556          * whether used as inputs or outputs, are set by boot code after
7557          * reset.
7558          */
7559         if (!(tp->tg3_flags2 & TG3_FLG2_IS_NIC)) {
7560                 u32 gpio_mask;
7561
7562                 gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
7563                             GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
7564                             GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
7565
7566                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
7567                         gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
7568                                      GRC_LCLCTRL_GPIO_OUTPUT3;
7569
7570                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
7571                         gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
7572
7573                 tp->grc_local_ctrl &= ~gpio_mask;
7574                 tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
7575
7576                 /* GPIO1 must be driven high for eeprom write protect */
7577                 if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT)
7578                         tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
7579                                                GRC_LCLCTRL_GPIO_OUTPUT1);
7580         }
7581         tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
7582         udelay(100);
7583
7584         if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX) {
7585                 val = tr32(MSGINT_MODE);
7586                 val |= MSGINT_MODE_MULTIVEC_EN | MSGINT_MODE_ENABLE;
7587                 tw32(MSGINT_MODE, val);
7588         }
7589
7590         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
7591                 tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
7592                 udelay(40);
7593         }
7594
7595         val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
7596                WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
7597                WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
7598                WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
7599                WDMAC_MODE_LNGREAD_ENAB);
7600
7601         /* If statement applies to 5705 and 5750 PCI devices only */
7602         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
7603              tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) ||
7604             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) {
7605                 if ((tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) &&
7606                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
7607                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
7608                         /* nothing */
7609                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
7610                            !(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
7611                            !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
7612                         val |= WDMAC_MODE_RX_ACCEL;
7613                 }
7614         }
7615
7616         /* Enable host coalescing bug fix */
7617         if (tp->tg3_flags3 & TG3_FLG3_5755_PLUS)
7618                 val |= WDMAC_MODE_STATUS_TAG_FIX;
7619
7620         tw32_f(WDMAC_MODE, val);
7621         udelay(40);
7622
7623         if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
7624                 u16 pcix_cmd;
7625
7626                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
7627                                      &pcix_cmd);
7628                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703) {
7629                         pcix_cmd &= ~PCI_X_CMD_MAX_READ;
7630                         pcix_cmd |= PCI_X_CMD_READ_2K;
7631                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
7632                         pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
7633                         pcix_cmd |= PCI_X_CMD_READ_2K;
7634                 }
7635                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
7636                                       pcix_cmd);
7637         }
7638
7639         tw32_f(RDMAC_MODE, rdmac_mode);
7640         udelay(40);
7641
7642         tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
7643         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
7644                 tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
7645
7646         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
7647                 tw32(SNDDATAC_MODE,
7648                      SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
7649         else
7650                 tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
7651
7652         tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
7653         tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
7654         tw32(RCVDBDI_MODE, RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ);
7655         tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
7656         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO)
7657                 tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
7658         val = SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE;
7659         if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX)
7660                 val |= SNDBDI_MODE_MULTI_TXQ_EN;
7661         tw32(SNDBDI_MODE, val);
7662         tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
7663
7664         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
7665                 err = tg3_load_5701_a0_firmware_fix(tp);
7666                 if (err)
7667                         return err;
7668         }
7669
7670         if (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) {
7671                 err = tg3_load_tso_firmware(tp);
7672                 if (err)
7673                         return err;
7674         }
7675
7676         tp->tx_mode = TX_MODE_ENABLE;
7677         tw32_f(MAC_TX_MODE, tp->tx_mode);
7678         udelay(100);
7679
7680         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_RSS) {
7681                 u32 reg = MAC_RSS_INDIR_TBL_0;
7682                 u8 *ent = (u8 *)&val;
7683
7684                 /* Setup the indirection table */
7685                 for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++) {
7686                         int idx = i % sizeof(val);
7687
7688                         ent[idx] = i % (tp->irq_cnt - 1);
7689                         if (idx == sizeof(val) - 1) {
7690                                 tw32(reg, val);
7691                                 reg += 4;
7692                         }
7693                 }
7694
7695                 /* Setup the "secret" hash key. */
7696                 tw32(MAC_RSS_HASH_KEY_0, 0x5f865437);
7697                 tw32(MAC_RSS_HASH_KEY_1, 0xe4ac62cc);
7698                 tw32(MAC_RSS_HASH_KEY_2, 0x50103a45);
7699                 tw32(MAC_RSS_HASH_KEY_3, 0x36621985);
7700                 tw32(MAC_RSS_HASH_KEY_4, 0xbf14c0e8);
7701                 tw32(MAC_RSS_HASH_KEY_5, 0x1bc27a1e);
7702                 tw32(MAC_RSS_HASH_KEY_6, 0x84f4b556);
7703                 tw32(MAC_RSS_HASH_KEY_7, 0x094ea6fe);
7704                 tw32(MAC_RSS_HASH_KEY_8, 0x7dda01e7);
7705                 tw32(MAC_RSS_HASH_KEY_9, 0xc04d7481);
7706         }
7707
7708         tp->rx_mode = RX_MODE_ENABLE;
7709         if (tp->tg3_flags3 & TG3_FLG3_5755_PLUS)
7710                 tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
7711
7712         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_RSS)
7713                 tp->rx_mode |= RX_MODE_RSS_ENABLE |
7714                                RX_MODE_RSS_ITBL_HASH_BITS_7 |
7715                                RX_MODE_RSS_IPV6_HASH_EN |
7716                                RX_MODE_RSS_TCP_IPV6_HASH_EN |
7717                                RX_MODE_RSS_IPV4_HASH_EN |
7718                                RX_MODE_RSS_TCP_IPV4_HASH_EN;
7719
7720         tw32_f(MAC_RX_MODE, tp->rx_mode);
7721         udelay(10);
7722
7723         tw32(MAC_LED_CTRL, tp->led_ctrl);
7724
7725         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
7726         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
7727                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
7728                 udelay(10);
7729         }
7730         tw32_f(MAC_RX_MODE, tp->rx_mode);
7731         udelay(10);
7732
7733         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
7734                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
7735                         !(tp->tg3_flags2 & TG3_FLG2_SERDES_PREEMPHASIS)) {
7736                         /* Set drive transmission level to 1.2V  */
7737                         /* only if the signal pre-emphasis bit is not set  */
7738                         val = tr32(MAC_SERDES_CFG);
7739                         val &= 0xfffff000;
7740                         val |= 0x880;
7741                         tw32(MAC_SERDES_CFG, val);
7742                 }
7743                 if (tp->pci_chip_rev_id == CHIPREV_ID_5703_A1)
7744                         tw32(MAC_SERDES_CFG, 0x616000);
7745         }
7746
7747         /* Prevent chip from dropping frames when flow control
7748          * is enabled.
7749          */
7750         tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, 2);
7751
7752         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
7753             (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
7754                 /* Use hardware link auto-negotiation */
7755                 tp->tg3_flags2 |= TG3_FLG2_HW_AUTONEG;
7756         }
7757
7758         if ((tp->tg3_flags2 & TG3_FLG2_MII_SERDES) &&
7759             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
7760                 u32 tmp;
7761
7762                 tmp = tr32(SERDES_RX_CTRL);
7763                 tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
7764                 tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
7765                 tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
7766                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
7767         }
7768
7769         if (!(tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB)) {
7770                 if (tp->link_config.phy_is_low_power) {
7771                         tp->link_config.phy_is_low_power = 0;
7772                         tp->link_config.speed = tp->link_config.orig_speed;
7773                         tp->link_config.duplex = tp->link_config.orig_duplex;
7774                         tp->link_config.autoneg = tp->link_config.orig_autoneg;
7775                 }
7776
7777                 err = tg3_setup_phy(tp, 0);
7778                 if (err)
7779                         return err;
7780
7781                 if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
7782                     !(tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET)) {
7783                         u32 tmp;
7784
7785                         /* Clear CRC stats. */
7786                         if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
7787                                 tg3_writephy(tp, MII_TG3_TEST1,
7788                                              tmp | MII_TG3_TEST1_CRC_EN);
7789                                 tg3_readphy(tp, 0x14, &tmp);
7790                         }
7791                 }
7792         }
7793
7794         __tg3_set_rx_mode(tp->dev);
7795
7796         /* Initialize receive rules. */
7797         tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
7798         tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
7799         tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
7800         tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
7801
7802         if ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
7803             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
7804                 limit = 8;
7805         else
7806                 limit = 16;
7807         if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF)
7808                 limit -= 4;
7809         switch (limit) {
7810         case 16:
7811                 tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
7812         case 15:
7813                 tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
7814         case 14:
7815                 tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
7816         case 13:
7817                 tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
7818         case 12:
7819                 tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
7820         case 11:
7821                 tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
7822         case 10:
7823                 tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
7824         case 9:
7825                 tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
7826         case 8:
7827                 tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
7828         case 7:
7829                 tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
7830         case 6:
7831                 tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
7832         case 5:
7833                 tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
7834         case 4:
7835                 /* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
7836         case 3:
7837                 /* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
7838         case 2:
7839         case 1:
7840
7841         default:
7842                 break;
7843         }
7844
7845         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
7846                 /* Write our heartbeat update interval to APE. */
7847                 tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
7848                                 APE_HOST_HEARTBEAT_INT_DISABLE);
7849
7850         tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
7851
7852         return 0;
7853 }
7854
7855 /* Called at device open time to get the chip ready for
7856  * packet processing.  Invoked with tp->lock held.
7857  */
7858 static int tg3_init_hw(struct tg3 *tp, int reset_phy)
7859 {
7860         tg3_switch_clocks(tp);
7861
7862         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
7863
7864         return tg3_reset_hw(tp, reset_phy);
7865 }
7866
7867 #define TG3_STAT_ADD32(PSTAT, REG) \
7868 do {    u32 __val = tr32(REG); \
7869         (PSTAT)->low += __val; \
7870         if ((PSTAT)->low < __val) \
7871                 (PSTAT)->high += 1; \
7872 } while (0)
7873
7874 static void tg3_periodic_fetch_stats(struct tg3 *tp)
7875 {
7876         struct tg3_hw_stats *sp = tp->hw_stats;
7877
7878         if (!netif_carrier_ok(tp->dev))
7879                 return;
7880
7881         TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
7882         TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
7883         TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
7884         TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
7885         TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
7886         TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
7887         TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
7888         TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
7889         TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
7890         TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
7891         TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
7892         TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
7893         TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
7894
7895         TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
7896         TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
7897         TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
7898         TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
7899         TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
7900         TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
7901         TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
7902         TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
7903         TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
7904         TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
7905         TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
7906         TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
7907         TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
7908         TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
7909
7910         TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
7911         TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
7912         TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
7913 }
7914
7915 static void tg3_timer(unsigned long __opaque)
7916 {
7917         struct tg3 *tp = (struct tg3 *) __opaque;
7918
7919         if (tp->irq_sync)
7920                 goto restart_timer;
7921
7922         spin_lock(&tp->lock);
7923
7924         if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
7925                 /* All of this garbage is because when using non-tagged
7926                  * IRQ status the mailbox/status_block protocol the chip
7927                  * uses with the cpu is race prone.
7928                  */
7929                 if (tp->napi[0].hw_status->status & SD_STATUS_UPDATED) {
7930                         tw32(GRC_LOCAL_CTRL,
7931                              tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
7932                 } else {
7933                         tw32(HOSTCC_MODE, tp->coalesce_mode |
7934                              HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW);
7935                 }
7936
7937                 if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
7938                         tp->tg3_flags2 |= TG3_FLG2_RESTART_TIMER;
7939                         spin_unlock(&tp->lock);
7940                         schedule_work(&tp->reset_task);
7941                         return;
7942                 }
7943         }
7944
7945         /* This part only runs once per second. */
7946         if (!--tp->timer_counter) {
7947                 if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
7948                         tg3_periodic_fetch_stats(tp);
7949
7950                 if (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) {
7951                         u32 mac_stat;
7952                         int phy_event;
7953
7954                         mac_stat = tr32(MAC_STATUS);
7955
7956                         phy_event = 0;
7957                         if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) {
7958                                 if (mac_stat & MAC_STATUS_MI_INTERRUPT)
7959                                         phy_event = 1;
7960                         } else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
7961                                 phy_event = 1;
7962
7963                         if (phy_event)
7964                                 tg3_setup_phy(tp, 0);
7965                 } else if (tp->tg3_flags & TG3_FLAG_POLL_SERDES) {
7966                         u32 mac_stat = tr32(MAC_STATUS);
7967                         int need_setup = 0;
7968
7969                         if (netif_carrier_ok(tp->dev) &&
7970                             (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
7971                                 need_setup = 1;
7972                         }
7973                         if (! netif_carrier_ok(tp->dev) &&
7974                             (mac_stat & (MAC_STATUS_PCS_SYNCED |
7975                                          MAC_STATUS_SIGNAL_DET))) {
7976                                 need_setup = 1;
7977                         }
7978                         if (need_setup) {
7979                                 if (!tp->serdes_counter) {
7980                                         tw32_f(MAC_MODE,
7981                                              (tp->mac_mode &
7982                                               ~MAC_MODE_PORT_MODE_MASK));
7983                                         udelay(40);
7984                                         tw32_f(MAC_MODE, tp->mac_mode);
7985                                         udelay(40);
7986                                 }
7987                                 tg3_setup_phy(tp, 0);
7988                         }
7989                 } else if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
7990                         tg3_serdes_parallel_detect(tp);
7991
7992                 tp->timer_counter = tp->timer_multiplier;
7993         }
7994
7995         /* Heartbeat is only sent once every 2 seconds.
7996          *
7997          * The heartbeat is to tell the ASF firmware that the host
7998          * driver is still alive.  In the event that the OS crashes,
7999          * ASF needs to reset the hardware to free up the FIFO space
8000          * that may be filled with rx packets destined for the host.
8001          * If the FIFO is full, ASF will no longer function properly.
8002          *
8003          * Unintended resets have been reported on real time kernels
8004          * where the timer doesn't run on time.  Netpoll will also have
8005          * same problem.
8006          *
8007          * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
8008          * to check the ring condition when the heartbeat is expiring
8009          * before doing the reset.  This will prevent most unintended
8010          * resets.
8011          */
8012         if (!--tp->asf_counter) {
8013                 if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) &&
8014                     !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
8015                         tg3_wait_for_event_ack(tp);
8016
8017                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
8018                                       FWCMD_NICDRV_ALIVE3);
8019                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
8020                         /* 5 seconds timeout */
8021                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, 5);
8022
8023                         tg3_generate_fw_event(tp);
8024                 }
8025                 tp->asf_counter = tp->asf_multiplier;
8026         }
8027
8028         spin_unlock(&tp->lock);
8029
8030 restart_timer:
8031         tp->timer.expires = jiffies + tp->timer_offset;
8032         add_timer(&tp->timer);
8033 }
8034
8035 static int tg3_request_irq(struct tg3 *tp, int irq_num)
8036 {
8037         irq_handler_t fn;
8038         unsigned long flags;
8039         char *name;
8040         struct tg3_napi *tnapi = &tp->napi[irq_num];
8041
8042         if (tp->irq_cnt == 1)
8043                 name = tp->dev->name;
8044         else {
8045                 name = &tnapi->irq_lbl[0];
8046                 snprintf(name, IFNAMSIZ, "%s-%d", tp->dev->name, irq_num);
8047                 name[IFNAMSIZ-1] = 0;
8048         }
8049
8050         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI_OR_MSIX) {
8051                 fn = tg3_msi;
8052                 if (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)
8053                         fn = tg3_msi_1shot;
8054                 flags = IRQF_SAMPLE_RANDOM;
8055         } else {
8056                 fn = tg3_interrupt;
8057                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
8058                         fn = tg3_interrupt_tagged;
8059                 flags = IRQF_SHARED | IRQF_SAMPLE_RANDOM;
8060         }
8061
8062         return request_irq(tnapi->irq_vec, fn, flags, name, tnapi);
8063 }
8064
8065 static int tg3_test_interrupt(struct tg3 *tp)
8066 {
8067         struct tg3_napi *tnapi = &tp->napi[0];
8068         struct net_device *dev = tp->dev;
8069         int err, i, intr_ok = 0;
8070         u32 val;
8071
8072         if (!netif_running(dev))
8073                 return -ENODEV;
8074
8075         tg3_disable_ints(tp);
8076
8077         free_irq(tnapi->irq_vec, tnapi);
8078
8079         /*
8080          * Turn off MSI one shot mode.  Otherwise this test has no
8081          * observable way to know whether the interrupt was delivered.
8082          */
8083         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 &&
8084             (tp->tg3_flags2 & TG3_FLG2_USING_MSI)) {
8085                 val = tr32(MSGINT_MODE) | MSGINT_MODE_ONE_SHOT_DISABLE;
8086                 tw32(MSGINT_MODE, val);
8087         }
8088
8089         err = request_irq(tnapi->irq_vec, tg3_test_isr,
8090                           IRQF_SHARED | IRQF_SAMPLE_RANDOM, dev->name, tnapi);
8091         if (err)
8092                 return err;
8093
8094         tnapi->hw_status->status &= ~SD_STATUS_UPDATED;
8095         tg3_enable_ints(tp);
8096
8097         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
8098                tnapi->coal_now);
8099
8100         for (i = 0; i < 5; i++) {
8101                 u32 int_mbox, misc_host_ctrl;
8102
8103                 int_mbox = tr32_mailbox(tnapi->int_mbox);
8104                 misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
8105
8106                 if ((int_mbox != 0) ||
8107                     (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
8108                         intr_ok = 1;
8109                         break;
8110                 }
8111
8112                 msleep(10);
8113         }
8114
8115         tg3_disable_ints(tp);
8116
8117         free_irq(tnapi->irq_vec, tnapi);
8118
8119         err = tg3_request_irq(tp, 0);
8120
8121         if (err)
8122                 return err;
8123
8124         if (intr_ok) {
8125                 /* Reenable MSI one shot mode. */
8126                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 &&
8127                     (tp->tg3_flags2 & TG3_FLG2_USING_MSI)) {
8128                         val = tr32(MSGINT_MODE) & ~MSGINT_MODE_ONE_SHOT_DISABLE;
8129                         tw32(MSGINT_MODE, val);
8130                 }
8131                 return 0;
8132         }
8133
8134         return -EIO;
8135 }
8136
8137 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
8138  * successfully restored
8139  */
8140 static int tg3_test_msi(struct tg3 *tp)
8141 {
8142         int err;
8143         u16 pci_cmd;
8144
8145         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSI))
8146                 return 0;
8147
8148         /* Turn off SERR reporting in case MSI terminates with Master
8149          * Abort.
8150          */
8151         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
8152         pci_write_config_word(tp->pdev, PCI_COMMAND,
8153                               pci_cmd & ~PCI_COMMAND_SERR);
8154
8155         err = tg3_test_interrupt(tp);
8156
8157         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
8158
8159         if (!err)
8160                 return 0;
8161
8162         /* other failures */
8163         if (err != -EIO)
8164                 return err;
8165
8166         /* MSI test failed, go back to INTx mode */
8167         printk(KERN_WARNING PFX "%s: No interrupt was generated using MSI, "
8168                "switching to INTx mode. Please report this failure to "
8169                "the PCI maintainer and include system chipset information.\n",
8170                        tp->dev->name);
8171
8172         free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
8173
8174         pci_disable_msi(tp->pdev);
8175
8176         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI;
8177
8178         err = tg3_request_irq(tp, 0);
8179         if (err)
8180                 return err;
8181
8182         /* Need to reset the chip because the MSI cycle may have terminated
8183          * with Master Abort.
8184          */
8185         tg3_full_lock(tp, 1);
8186
8187         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8188         err = tg3_init_hw(tp, 1);
8189
8190         tg3_full_unlock(tp);
8191
8192         if (err)
8193                 free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
8194
8195         return err;
8196 }
8197
8198 static int tg3_request_firmware(struct tg3 *tp)
8199 {
8200         const __be32 *fw_data;
8201
8202         if (request_firmware(&tp->fw, tp->fw_needed, &tp->pdev->dev)) {
8203                 printk(KERN_ERR "%s: Failed to load firmware \"%s\"\n",
8204                        tp->dev->name, tp->fw_needed);
8205                 return -ENOENT;
8206         }
8207
8208         fw_data = (void *)tp->fw->data;
8209
8210         /* Firmware blob starts with version numbers, followed by
8211          * start address and _full_ length including BSS sections
8212          * (which must be longer than the actual data, of course
8213          */
8214
8215         tp->fw_len = be32_to_cpu(fw_data[2]);   /* includes bss */
8216         if (tp->fw_len < (tp->fw->size - 12)) {
8217                 printk(KERN_ERR "%s: bogus length %d in \"%s\"\n",
8218                        tp->dev->name, tp->fw_len, tp->fw_needed);
8219                 release_firmware(tp->fw);
8220                 tp->fw = NULL;
8221                 return -EINVAL;
8222         }
8223
8224         /* We no longer need firmware; we have it. */
8225         tp->fw_needed = NULL;
8226         return 0;
8227 }
8228
8229 static bool tg3_enable_msix(struct tg3 *tp)
8230 {
8231         int i, rc, cpus = num_online_cpus();
8232         struct msix_entry msix_ent[tp->irq_max];
8233
8234         if (cpus == 1)
8235                 /* Just fallback to the simpler MSI mode. */
8236                 return false;
8237
8238         /*
8239          * We want as many rx rings enabled as there are cpus.
8240          * The first MSIX vector only deals with link interrupts, etc,
8241          * so we add one to the number of vectors we are requesting.
8242          */
8243         tp->irq_cnt = min_t(unsigned, cpus + 1, tp->irq_max);
8244
8245         for (i = 0; i < tp->irq_max; i++) {
8246                 msix_ent[i].entry  = i;
8247                 msix_ent[i].vector = 0;
8248         }
8249
8250         rc = pci_enable_msix(tp->pdev, msix_ent, tp->irq_cnt);
8251         if (rc != 0) {
8252                 if (rc < TG3_RSS_MIN_NUM_MSIX_VECS)
8253                         return false;
8254                 if (pci_enable_msix(tp->pdev, msix_ent, rc))
8255                         return false;
8256                 printk(KERN_NOTICE
8257                        "%s: Requested %d MSI-X vectors, received %d\n",
8258                        tp->dev->name, tp->irq_cnt, rc);
8259                 tp->irq_cnt = rc;
8260         }
8261
8262         tp->tg3_flags3 |= TG3_FLG3_ENABLE_RSS;
8263
8264         for (i = 0; i < tp->irq_max; i++)
8265                 tp->napi[i].irq_vec = msix_ent[i].vector;
8266
8267         tp->dev->real_num_tx_queues = tp->irq_cnt - 1;
8268
8269         return true;
8270 }
8271
8272 static void tg3_ints_init(struct tg3 *tp)
8273 {
8274         if ((tp->tg3_flags & TG3_FLAG_SUPPORT_MSI_OR_MSIX) &&
8275             !(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) {
8276                 /* All MSI supporting chips should support tagged
8277                  * status.  Assert that this is the case.
8278                  */
8279                 printk(KERN_WARNING PFX "%s: MSI without TAGGED? "
8280                        "Not using MSI.\n", tp->dev->name);
8281                 goto defcfg;
8282         }
8283
8284         if ((tp->tg3_flags & TG3_FLAG_SUPPORT_MSIX) && tg3_enable_msix(tp))
8285                 tp->tg3_flags2 |= TG3_FLG2_USING_MSIX;
8286         else if ((tp->tg3_flags & TG3_FLAG_SUPPORT_MSI) &&
8287                  pci_enable_msi(tp->pdev) == 0)
8288                 tp->tg3_flags2 |= TG3_FLG2_USING_MSI;
8289
8290         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI_OR_MSIX) {
8291                 u32 msi_mode = tr32(MSGINT_MODE);
8292                 if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX)
8293                         msi_mode |= MSGINT_MODE_MULTIVEC_EN;
8294                 tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
8295         }
8296 defcfg:
8297         if (!(tp->tg3_flags2 & TG3_FLG2_USING_MSIX)) {
8298                 tp->irq_cnt = 1;
8299                 tp->napi[0].irq_vec = tp->pdev->irq;
8300                 tp->dev->real_num_tx_queues = 1;
8301         }
8302 }
8303
8304 static void tg3_ints_fini(struct tg3 *tp)
8305 {
8306         if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX)
8307                 pci_disable_msix(tp->pdev);
8308         else if (tp->tg3_flags2 & TG3_FLG2_USING_MSI)
8309                 pci_disable_msi(tp->pdev);
8310         tp->tg3_flags2 &= ~TG3_FLG2_USING_MSI_OR_MSIX;
8311         tp->tg3_flags3 &= ~TG3_FLG3_ENABLE_RSS;
8312 }
8313
8314 static int tg3_open(struct net_device *dev)
8315 {
8316         struct tg3 *tp = netdev_priv(dev);
8317         int i, err;
8318
8319         if (tp->fw_needed) {
8320                 err = tg3_request_firmware(tp);
8321                 if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
8322                         if (err)
8323                                 return err;
8324                 } else if (err) {
8325                         printk(KERN_WARNING "%s: TSO capability disabled.\n",
8326                                tp->dev->name);
8327                         tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
8328                 } else if (!(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
8329                         printk(KERN_NOTICE "%s: TSO capability restored.\n",
8330                                tp->dev->name);
8331                         tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
8332                 }
8333         }
8334
8335         netif_carrier_off(tp->dev);
8336
8337         err = tg3_set_power_state(tp, PCI_D0);
8338         if (err)
8339                 return err;
8340
8341         tg3_full_lock(tp, 0);
8342
8343         tg3_disable_ints(tp);
8344         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
8345
8346         tg3_full_unlock(tp);
8347
8348         /*
8349          * Setup interrupts first so we know how
8350          * many NAPI resources to allocate
8351          */
8352         tg3_ints_init(tp);
8353
8354         /* The placement of this call is tied
8355          * to the setup and use of Host TX descriptors.
8356          */
8357         err = tg3_alloc_consistent(tp);
8358         if (err)
8359                 goto err_out1;
8360
8361         tg3_napi_enable(tp);
8362
8363         for (i = 0; i < tp->irq_cnt; i++) {
8364                 struct tg3_napi *tnapi = &tp->napi[i];
8365                 err = tg3_request_irq(tp, i);
8366                 if (err) {
8367                         for (i--; i >= 0; i--)
8368                                 free_irq(tnapi->irq_vec, tnapi);
8369                         break;
8370                 }
8371         }
8372
8373         if (err)
8374                 goto err_out2;
8375
8376         tg3_full_lock(tp, 0);
8377
8378         err = tg3_init_hw(tp, 1);
8379         if (err) {
8380                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8381                 tg3_free_rings(tp);
8382         } else {
8383                 if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)
8384                         tp->timer_offset = HZ;
8385                 else
8386                         tp->timer_offset = HZ / 10;
8387
8388                 BUG_ON(tp->timer_offset > HZ);
8389                 tp->timer_counter = tp->timer_multiplier =
8390                         (HZ / tp->timer_offset);
8391                 tp->asf_counter = tp->asf_multiplier =
8392                         ((HZ / tp->timer_offset) * 2);
8393
8394                 init_timer(&tp->timer);
8395                 tp->timer.expires = jiffies + tp->timer_offset;
8396                 tp->timer.data = (unsigned long) tp;
8397                 tp->timer.function = tg3_timer;
8398         }
8399
8400         tg3_full_unlock(tp);
8401
8402         if (err)
8403                 goto err_out3;
8404
8405         if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) {
8406                 err = tg3_test_msi(tp);
8407
8408                 if (err) {
8409                         tg3_full_lock(tp, 0);
8410                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8411                         tg3_free_rings(tp);
8412                         tg3_full_unlock(tp);
8413
8414                         goto err_out2;
8415                 }
8416
8417                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717 &&
8418                     (tp->tg3_flags2 & TG3_FLG2_USING_MSI) &&
8419                     (tp->tg3_flags2 & TG3_FLG2_1SHOT_MSI)) {
8420                         u32 val = tr32(PCIE_TRANSACTION_CFG);
8421
8422                         tw32(PCIE_TRANSACTION_CFG,
8423                              val | PCIE_TRANS_CFG_1SHOT_MSI);
8424                 }
8425         }
8426
8427         tg3_phy_start(tp);
8428
8429         tg3_full_lock(tp, 0);
8430
8431         add_timer(&tp->timer);
8432         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
8433         tg3_enable_ints(tp);
8434
8435         tg3_full_unlock(tp);
8436
8437         netif_tx_start_all_queues(dev);
8438
8439         return 0;
8440
8441 err_out3:
8442         for (i = tp->irq_cnt - 1; i >= 0; i--) {
8443                 struct tg3_napi *tnapi = &tp->napi[i];
8444                 free_irq(tnapi->irq_vec, tnapi);
8445         }
8446
8447 err_out2:
8448         tg3_napi_disable(tp);
8449         tg3_free_consistent(tp);
8450
8451 err_out1:
8452         tg3_ints_fini(tp);
8453         return err;
8454 }
8455
8456 #if 0
8457 /*static*/ void tg3_dump_state(struct tg3 *tp)
8458 {
8459         u32 val32, val32_2, val32_3, val32_4, val32_5;
8460         u16 val16;
8461         int i;
8462         struct tg3_hw_status *sblk = tp->napi[0]->hw_status;
8463
8464         pci_read_config_word(tp->pdev, PCI_STATUS, &val16);
8465         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE, &val32);
8466         printk("DEBUG: PCI status [%04x] TG3PCI state[%08x]\n",
8467                val16, val32);
8468
8469         /* MAC block */
8470         printk("DEBUG: MAC_MODE[%08x] MAC_STATUS[%08x]\n",
8471                tr32(MAC_MODE), tr32(MAC_STATUS));
8472         printk("       MAC_EVENT[%08x] MAC_LED_CTRL[%08x]\n",
8473                tr32(MAC_EVENT), tr32(MAC_LED_CTRL));
8474         printk("DEBUG: MAC_TX_MODE[%08x] MAC_TX_STATUS[%08x]\n",
8475                tr32(MAC_TX_MODE), tr32(MAC_TX_STATUS));
8476         printk("       MAC_RX_MODE[%08x] MAC_RX_STATUS[%08x]\n",
8477                tr32(MAC_RX_MODE), tr32(MAC_RX_STATUS));
8478
8479         /* Send data initiator control block */
8480         printk("DEBUG: SNDDATAI_MODE[%08x] SNDDATAI_STATUS[%08x]\n",
8481                tr32(SNDDATAI_MODE), tr32(SNDDATAI_STATUS));
8482         printk("       SNDDATAI_STATSCTRL[%08x]\n",
8483                tr32(SNDDATAI_STATSCTRL));
8484
8485         /* Send data completion control block */
8486         printk("DEBUG: SNDDATAC_MODE[%08x]\n", tr32(SNDDATAC_MODE));
8487
8488         /* Send BD ring selector block */
8489         printk("DEBUG: SNDBDS_MODE[%08x] SNDBDS_STATUS[%08x]\n",
8490                tr32(SNDBDS_MODE), tr32(SNDBDS_STATUS));
8491
8492         /* Send BD initiator control block */
8493         printk("DEBUG: SNDBDI_MODE[%08x] SNDBDI_STATUS[%08x]\n",
8494                tr32(SNDBDI_MODE), tr32(SNDBDI_STATUS));
8495
8496         /* Send BD completion control block */
8497         printk("DEBUG: SNDBDC_MODE[%08x]\n", tr32(SNDBDC_MODE));
8498
8499         /* Receive list placement control block */
8500         printk("DEBUG: RCVLPC_MODE[%08x] RCVLPC_STATUS[%08x]\n",
8501                tr32(RCVLPC_MODE), tr32(RCVLPC_STATUS));
8502         printk("       RCVLPC_STATSCTRL[%08x]\n",
8503                tr32(RCVLPC_STATSCTRL));
8504
8505         /* Receive data and receive BD initiator control block */
8506         printk("DEBUG: RCVDBDI_MODE[%08x] RCVDBDI_STATUS[%08x]\n",
8507                tr32(RCVDBDI_MODE), tr32(RCVDBDI_STATUS));
8508
8509         /* Receive data completion control block */
8510         printk("DEBUG: RCVDCC_MODE[%08x]\n",
8511                tr32(RCVDCC_MODE));
8512
8513         /* Receive BD initiator control block */
8514         printk("DEBUG: RCVBDI_MODE[%08x] RCVBDI_STATUS[%08x]\n",
8515                tr32(RCVBDI_MODE), tr32(RCVBDI_STATUS));
8516
8517         /* Receive BD completion control block */
8518         printk("DEBUG: RCVCC_MODE[%08x] RCVCC_STATUS[%08x]\n",
8519                tr32(RCVCC_MODE), tr32(RCVCC_STATUS));
8520
8521         /* Receive list selector control block */
8522         printk("DEBUG: RCVLSC_MODE[%08x] RCVLSC_STATUS[%08x]\n",
8523                tr32(RCVLSC_MODE), tr32(RCVLSC_STATUS));
8524
8525         /* Mbuf cluster free block */
8526         printk("DEBUG: MBFREE_MODE[%08x] MBFREE_STATUS[%08x]\n",
8527                tr32(MBFREE_MODE), tr32(MBFREE_STATUS));
8528
8529         /* Host coalescing control block */
8530         printk("DEBUG: HOSTCC_MODE[%08x] HOSTCC_STATUS[%08x]\n",
8531                tr32(HOSTCC_MODE), tr32(HOSTCC_STATUS));
8532         printk("DEBUG: HOSTCC_STATS_BLK_HOST_ADDR[%08x%08x]\n",
8533                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
8534                tr32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
8535         printk("DEBUG: HOSTCC_STATUS_BLK_HOST_ADDR[%08x%08x]\n",
8536                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH),
8537                tr32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW));
8538         printk("DEBUG: HOSTCC_STATS_BLK_NIC_ADDR[%08x]\n",
8539                tr32(HOSTCC_STATS_BLK_NIC_ADDR));
8540         printk("DEBUG: HOSTCC_STATUS_BLK_NIC_ADDR[%08x]\n",
8541                tr32(HOSTCC_STATUS_BLK_NIC_ADDR));
8542
8543         /* Memory arbiter control block */
8544         printk("DEBUG: MEMARB_MODE[%08x] MEMARB_STATUS[%08x]\n",
8545                tr32(MEMARB_MODE), tr32(MEMARB_STATUS));
8546
8547         /* Buffer manager control block */
8548         printk("DEBUG: BUFMGR_MODE[%08x] BUFMGR_STATUS[%08x]\n",
8549                tr32(BUFMGR_MODE), tr32(BUFMGR_STATUS));
8550         printk("DEBUG: BUFMGR_MB_POOL_ADDR[%08x] BUFMGR_MB_POOL_SIZE[%08x]\n",
8551                tr32(BUFMGR_MB_POOL_ADDR), tr32(BUFMGR_MB_POOL_SIZE));
8552         printk("DEBUG: BUFMGR_DMA_DESC_POOL_ADDR[%08x] "
8553                "BUFMGR_DMA_DESC_POOL_SIZE[%08x]\n",
8554                tr32(BUFMGR_DMA_DESC_POOL_ADDR),
8555                tr32(BUFMGR_DMA_DESC_POOL_SIZE));
8556
8557         /* Read DMA control block */
8558         printk("DEBUG: RDMAC_MODE[%08x] RDMAC_STATUS[%08x]\n",
8559                tr32(RDMAC_MODE), tr32(RDMAC_STATUS));
8560
8561         /* Write DMA control block */
8562         printk("DEBUG: WDMAC_MODE[%08x] WDMAC_STATUS[%08x]\n",
8563                tr32(WDMAC_MODE), tr32(WDMAC_STATUS));
8564
8565         /* DMA completion block */
8566         printk("DEBUG: DMAC_MODE[%08x]\n",
8567                tr32(DMAC_MODE));
8568
8569         /* GRC block */
8570         printk("DEBUG: GRC_MODE[%08x] GRC_MISC_CFG[%08x]\n",
8571                tr32(GRC_MODE), tr32(GRC_MISC_CFG));
8572         printk("DEBUG: GRC_LOCAL_CTRL[%08x]\n",
8573                tr32(GRC_LOCAL_CTRL));
8574
8575         /* TG3_BDINFOs */
8576         printk("DEBUG: RCVDBDI_JUMBO_BD[%08x%08x:%08x:%08x]\n",
8577                tr32(RCVDBDI_JUMBO_BD + 0x0),
8578                tr32(RCVDBDI_JUMBO_BD + 0x4),
8579                tr32(RCVDBDI_JUMBO_BD + 0x8),
8580                tr32(RCVDBDI_JUMBO_BD + 0xc));
8581         printk("DEBUG: RCVDBDI_STD_BD[%08x%08x:%08x:%08x]\n",
8582                tr32(RCVDBDI_STD_BD + 0x0),
8583                tr32(RCVDBDI_STD_BD + 0x4),
8584                tr32(RCVDBDI_STD_BD + 0x8),
8585                tr32(RCVDBDI_STD_BD + 0xc));
8586         printk("DEBUG: RCVDBDI_MINI_BD[%08x%08x:%08x:%08x]\n",
8587                tr32(RCVDBDI_MINI_BD + 0x0),
8588                tr32(RCVDBDI_MINI_BD + 0x4),
8589                tr32(RCVDBDI_MINI_BD + 0x8),
8590                tr32(RCVDBDI_MINI_BD + 0xc));
8591
8592         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x0, &val32);
8593         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x4, &val32_2);
8594         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0x8, &val32_3);
8595         tg3_read_mem(tp, NIC_SRAM_SEND_RCB + 0xc, &val32_4);
8596         printk("DEBUG: SRAM_SEND_RCB_0[%08x%08x:%08x:%08x]\n",
8597                val32, val32_2, val32_3, val32_4);
8598
8599         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x0, &val32);
8600         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x4, &val32_2);
8601         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0x8, &val32_3);
8602         tg3_read_mem(tp, NIC_SRAM_RCV_RET_RCB + 0xc, &val32_4);
8603         printk("DEBUG: SRAM_RCV_RET_RCB_0[%08x%08x:%08x:%08x]\n",
8604                val32, val32_2, val32_3, val32_4);
8605
8606         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x0, &val32);
8607         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x4, &val32_2);
8608         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x8, &val32_3);
8609         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0xc, &val32_4);
8610         tg3_read_mem(tp, NIC_SRAM_STATUS_BLK + 0x10, &val32_5);
8611         printk("DEBUG: SRAM_STATUS_BLK[%08x:%08x:%08x:%08x:%08x]\n",
8612                val32, val32_2, val32_3, val32_4, val32_5);
8613
8614         /* SW status block */
8615         printk(KERN_DEBUG
8616          "Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
8617                sblk->status,
8618                sblk->status_tag,
8619                sblk->rx_jumbo_consumer,
8620                sblk->rx_consumer,
8621                sblk->rx_mini_consumer,
8622                sblk->idx[0].rx_producer,
8623                sblk->idx[0].tx_consumer);
8624
8625         /* SW statistics block */
8626         printk("DEBUG: Host statistics block [%08x:%08x:%08x:%08x]\n",
8627                ((u32 *)tp->hw_stats)[0],
8628                ((u32 *)tp->hw_stats)[1],
8629                ((u32 *)tp->hw_stats)[2],
8630                ((u32 *)tp->hw_stats)[3]);
8631
8632         /* Mailboxes */
8633         printk("DEBUG: SNDHOST_PROD[%08x%08x] SNDNIC_PROD[%08x%08x]\n",
8634                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x0),
8635                tr32_mailbox(MAILBOX_SNDHOST_PROD_IDX_0 + 0x4),
8636                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x0),
8637                tr32_mailbox(MAILBOX_SNDNIC_PROD_IDX_0 + 0x4));
8638
8639         /* NIC side send descriptors. */
8640         for (i = 0; i < 6; i++) {
8641                 unsigned long txd;
8642
8643                 txd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_TX_BUFFER_DESC
8644                         + (i * sizeof(struct tg3_tx_buffer_desc));
8645                 printk("DEBUG: NIC TXD(%d)[%08x:%08x:%08x:%08x]\n",
8646                        i,
8647                        readl(txd + 0x0), readl(txd + 0x4),
8648                        readl(txd + 0x8), readl(txd + 0xc));
8649         }
8650
8651         /* NIC side RX descriptors. */
8652         for (i = 0; i < 6; i++) {
8653                 unsigned long rxd;
8654
8655                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_BUFFER_DESC
8656                         + (i * sizeof(struct tg3_rx_buffer_desc));
8657                 printk("DEBUG: NIC RXD_STD(%d)[0][%08x:%08x:%08x:%08x]\n",
8658                        i,
8659                        readl(rxd + 0x0), readl(rxd + 0x4),
8660                        readl(rxd + 0x8), readl(rxd + 0xc));
8661                 rxd += (4 * sizeof(u32));
8662                 printk("DEBUG: NIC RXD_STD(%d)[1][%08x:%08x:%08x:%08x]\n",
8663                        i,
8664                        readl(rxd + 0x0), readl(rxd + 0x4),
8665                        readl(rxd + 0x8), readl(rxd + 0xc));
8666         }
8667
8668         for (i = 0; i < 6; i++) {
8669                 unsigned long rxd;
8670
8671                 rxd = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_RX_JUMBO_BUFFER_DESC
8672                         + (i * sizeof(struct tg3_rx_buffer_desc));
8673                 printk("DEBUG: NIC RXD_JUMBO(%d)[0][%08x:%08x:%08x:%08x]\n",
8674                        i,
8675                        readl(rxd + 0x0), readl(rxd + 0x4),
8676                        readl(rxd + 0x8), readl(rxd + 0xc));
8677                 rxd += (4 * sizeof(u32));
8678                 printk("DEBUG: NIC RXD_JUMBO(%d)[1][%08x:%08x:%08x:%08x]\n",
8679                        i,
8680                        readl(rxd + 0x0), readl(rxd + 0x4),
8681                        readl(rxd + 0x8), readl(rxd + 0xc));
8682         }
8683 }
8684 #endif
8685
8686 static struct net_device_stats *tg3_get_stats(struct net_device *);
8687 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *);
8688
8689 static int tg3_close(struct net_device *dev)
8690 {
8691         int i;
8692         struct tg3 *tp = netdev_priv(dev);
8693
8694         tg3_napi_disable(tp);
8695         cancel_work_sync(&tp->reset_task);
8696
8697         netif_tx_stop_all_queues(dev);
8698
8699         del_timer_sync(&tp->timer);
8700
8701         tg3_full_lock(tp, 1);
8702 #if 0
8703         tg3_dump_state(tp);
8704 #endif
8705
8706         tg3_disable_ints(tp);
8707
8708         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
8709         tg3_free_rings(tp);
8710         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
8711
8712         tg3_full_unlock(tp);
8713
8714         for (i = tp->irq_cnt - 1; i >= 0; i--) {
8715                 struct tg3_napi *tnapi = &tp->napi[i];
8716                 free_irq(tnapi->irq_vec, tnapi);
8717         }
8718
8719         tg3_ints_fini(tp);
8720
8721         memcpy(&tp->net_stats_prev, tg3_get_stats(tp->dev),
8722                sizeof(tp->net_stats_prev));
8723         memcpy(&tp->estats_prev, tg3_get_estats(tp),
8724                sizeof(tp->estats_prev));
8725
8726         tg3_free_consistent(tp);
8727
8728         tg3_set_power_state(tp, PCI_D3hot);
8729
8730         netif_carrier_off(tp->dev);
8731
8732         return 0;
8733 }
8734
8735 static inline unsigned long get_stat64(tg3_stat64_t *val)
8736 {
8737         unsigned long ret;
8738
8739 #if (BITS_PER_LONG == 32)
8740         ret = val->low;
8741 #else
8742         ret = ((u64)val->high << 32) | ((u64)val->low);
8743 #endif
8744         return ret;
8745 }
8746
8747 static inline u64 get_estat64(tg3_stat64_t *val)
8748 {
8749        return ((u64)val->high << 32) | ((u64)val->low);
8750 }
8751
8752 static unsigned long calc_crc_errors(struct tg3 *tp)
8753 {
8754         struct tg3_hw_stats *hw_stats = tp->hw_stats;
8755
8756         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
8757             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
8758              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
8759                 u32 val;
8760
8761                 spin_lock_bh(&tp->lock);
8762                 if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
8763                         tg3_writephy(tp, MII_TG3_TEST1,
8764                                      val | MII_TG3_TEST1_CRC_EN);
8765                         tg3_readphy(tp, 0x14, &val);
8766                 } else
8767                         val = 0;
8768                 spin_unlock_bh(&tp->lock);
8769
8770                 tp->phy_crc_errors += val;
8771
8772                 return tp->phy_crc_errors;
8773         }
8774
8775         return get_stat64(&hw_stats->rx_fcs_errors);
8776 }
8777
8778 #define ESTAT_ADD(member) \
8779         estats->member =        old_estats->member + \
8780                                 get_estat64(&hw_stats->member)
8781
8782 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *tp)
8783 {
8784         struct tg3_ethtool_stats *estats = &tp->estats;
8785         struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
8786         struct tg3_hw_stats *hw_stats = tp->hw_stats;
8787
8788         if (!hw_stats)
8789                 return old_estats;
8790
8791         ESTAT_ADD(rx_octets);
8792         ESTAT_ADD(rx_fragments);
8793         ESTAT_ADD(rx_ucast_packets);
8794         ESTAT_ADD(rx_mcast_packets);
8795         ESTAT_ADD(rx_bcast_packets);
8796         ESTAT_ADD(rx_fcs_errors);
8797         ESTAT_ADD(rx_align_errors);
8798         ESTAT_ADD(rx_xon_pause_rcvd);
8799         ESTAT_ADD(rx_xoff_pause_rcvd);
8800         ESTAT_ADD(rx_mac_ctrl_rcvd);
8801         ESTAT_ADD(rx_xoff_entered);
8802         ESTAT_ADD(rx_frame_too_long_errors);
8803         ESTAT_ADD(rx_jabbers);
8804         ESTAT_ADD(rx_undersize_packets);
8805         ESTAT_ADD(rx_in_length_errors);
8806         ESTAT_ADD(rx_out_length_errors);
8807         ESTAT_ADD(rx_64_or_less_octet_packets);
8808         ESTAT_ADD(rx_65_to_127_octet_packets);
8809         ESTAT_ADD(rx_128_to_255_octet_packets);
8810         ESTAT_ADD(rx_256_to_511_octet_packets);
8811         ESTAT_ADD(rx_512_to_1023_octet_packets);
8812         ESTAT_ADD(rx_1024_to_1522_octet_packets);
8813         ESTAT_ADD(rx_1523_to_2047_octet_packets);
8814         ESTAT_ADD(rx_2048_to_4095_octet_packets);
8815         ESTAT_ADD(rx_4096_to_8191_octet_packets);
8816         ESTAT_ADD(rx_8192_to_9022_octet_packets);
8817
8818         ESTAT_ADD(tx_octets);
8819         ESTAT_ADD(tx_collisions);
8820         ESTAT_ADD(tx_xon_sent);
8821         ESTAT_ADD(tx_xoff_sent);
8822         ESTAT_ADD(tx_flow_control);
8823         ESTAT_ADD(tx_mac_errors);
8824         ESTAT_ADD(tx_single_collisions);
8825         ESTAT_ADD(tx_mult_collisions);
8826         ESTAT_ADD(tx_deferred);
8827         ESTAT_ADD(tx_excessive_collisions);
8828         ESTAT_ADD(tx_late_collisions);
8829         ESTAT_ADD(tx_collide_2times);
8830         ESTAT_ADD(tx_collide_3times);
8831         ESTAT_ADD(tx_collide_4times);
8832         ESTAT_ADD(tx_collide_5times);
8833         ESTAT_ADD(tx_collide_6times);
8834         ESTAT_ADD(tx_collide_7times);
8835         ESTAT_ADD(tx_collide_8times);
8836         ESTAT_ADD(tx_collide_9times);
8837         ESTAT_ADD(tx_collide_10times);
8838         ESTAT_ADD(tx_collide_11times);
8839         ESTAT_ADD(tx_collide_12times);
8840         ESTAT_ADD(tx_collide_13times);
8841         ESTAT_ADD(tx_collide_14times);
8842         ESTAT_ADD(tx_collide_15times);
8843         ESTAT_ADD(tx_ucast_packets);
8844         ESTAT_ADD(tx_mcast_packets);
8845         ESTAT_ADD(tx_bcast_packets);
8846         ESTAT_ADD(tx_carrier_sense_errors);
8847         ESTAT_ADD(tx_discards);
8848         ESTAT_ADD(tx_errors);
8849
8850         ESTAT_ADD(dma_writeq_full);
8851         ESTAT_ADD(dma_write_prioq_full);
8852         ESTAT_ADD(rxbds_empty);
8853         ESTAT_ADD(rx_discards);
8854         ESTAT_ADD(rx_errors);
8855         ESTAT_ADD(rx_threshold_hit);
8856
8857         ESTAT_ADD(dma_readq_full);
8858         ESTAT_ADD(dma_read_prioq_full);
8859         ESTAT_ADD(tx_comp_queue_full);
8860
8861         ESTAT_ADD(ring_set_send_prod_index);
8862         ESTAT_ADD(ring_status_update);
8863         ESTAT_ADD(nic_irqs);
8864         ESTAT_ADD(nic_avoided_irqs);
8865         ESTAT_ADD(nic_tx_threshold_hit);
8866
8867         return estats;
8868 }
8869
8870 static struct net_device_stats *tg3_get_stats(struct net_device *dev)
8871 {
8872         struct tg3 *tp = netdev_priv(dev);
8873         struct net_device_stats *stats = &tp->net_stats;
8874         struct net_device_stats *old_stats = &tp->net_stats_prev;
8875         struct tg3_hw_stats *hw_stats = tp->hw_stats;
8876
8877         if (!hw_stats)
8878                 return old_stats;
8879
8880         stats->rx_packets = old_stats->rx_packets +
8881                 get_stat64(&hw_stats->rx_ucast_packets) +
8882                 get_stat64(&hw_stats->rx_mcast_packets) +
8883                 get_stat64(&hw_stats->rx_bcast_packets);
8884
8885         stats->tx_packets = old_stats->tx_packets +
8886                 get_stat64(&hw_stats->tx_ucast_packets) +
8887                 get_stat64(&hw_stats->tx_mcast_packets) +
8888                 get_stat64(&hw_stats->tx_bcast_packets);
8889
8890         stats->rx_bytes = old_stats->rx_bytes +
8891                 get_stat64(&hw_stats->rx_octets);
8892         stats->tx_bytes = old_stats->tx_bytes +
8893                 get_stat64(&hw_stats->tx_octets);
8894
8895         stats->rx_errors = old_stats->rx_errors +
8896                 get_stat64(&hw_stats->rx_errors);
8897         stats->tx_errors = old_stats->tx_errors +
8898                 get_stat64(&hw_stats->tx_errors) +
8899                 get_stat64(&hw_stats->tx_mac_errors) +
8900                 get_stat64(&hw_stats->tx_carrier_sense_errors) +
8901                 get_stat64(&hw_stats->tx_discards);
8902
8903         stats->multicast = old_stats->multicast +
8904                 get_stat64(&hw_stats->rx_mcast_packets);
8905         stats->collisions = old_stats->collisions +
8906                 get_stat64(&hw_stats->tx_collisions);
8907
8908         stats->rx_length_errors = old_stats->rx_length_errors +
8909                 get_stat64(&hw_stats->rx_frame_too_long_errors) +
8910                 get_stat64(&hw_stats->rx_undersize_packets);
8911
8912         stats->rx_over_errors = old_stats->rx_over_errors +
8913                 get_stat64(&hw_stats->rxbds_empty);
8914         stats->rx_frame_errors = old_stats->rx_frame_errors +
8915                 get_stat64(&hw_stats->rx_align_errors);
8916         stats->tx_aborted_errors = old_stats->tx_aborted_errors +
8917                 get_stat64(&hw_stats->tx_discards);
8918         stats->tx_carrier_errors = old_stats->tx_carrier_errors +
8919                 get_stat64(&hw_stats->tx_carrier_sense_errors);
8920
8921         stats->rx_crc_errors = old_stats->rx_crc_errors +
8922                 calc_crc_errors(tp);
8923
8924         stats->rx_missed_errors = old_stats->rx_missed_errors +
8925                 get_stat64(&hw_stats->rx_discards);
8926
8927         return stats;
8928 }
8929
8930 static inline u32 calc_crc(unsigned char *buf, int len)
8931 {
8932         u32 reg;
8933         u32 tmp;
8934         int j, k;
8935
8936         reg = 0xffffffff;
8937
8938         for (j = 0; j < len; j++) {
8939                 reg ^= buf[j];
8940
8941                 for (k = 0; k < 8; k++) {
8942                         tmp = reg & 0x01;
8943
8944                         reg >>= 1;
8945
8946                         if (tmp) {
8947                                 reg ^= 0xedb88320;
8948                         }
8949                 }
8950         }
8951
8952         return ~reg;
8953 }
8954
8955 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
8956 {
8957         /* accept or reject all multicast frames */
8958         tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
8959         tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
8960         tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
8961         tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
8962 }
8963
8964 static void __tg3_set_rx_mode(struct net_device *dev)
8965 {
8966         struct tg3 *tp = netdev_priv(dev);
8967         u32 rx_mode;
8968
8969         rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
8970                                   RX_MODE_KEEP_VLAN_TAG);
8971
8972         /* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
8973          * flag clear.
8974          */
8975 #if TG3_VLAN_TAG_USED
8976         if (!tp->vlgrp &&
8977             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8978                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8979 #else
8980         /* By definition, VLAN is disabled always in this
8981          * case.
8982          */
8983         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF))
8984                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
8985 #endif
8986
8987         if (dev->flags & IFF_PROMISC) {
8988                 /* Promiscuous mode. */
8989                 rx_mode |= RX_MODE_PROMISC;
8990         } else if (dev->flags & IFF_ALLMULTI) {
8991                 /* Accept all multicast. */
8992                 tg3_set_multi (tp, 1);
8993         } else if (dev->mc_count < 1) {
8994                 /* Reject all multicast. */
8995                 tg3_set_multi (tp, 0);
8996         } else {
8997                 /* Accept one or more multicast(s). */
8998                 struct dev_mc_list *mclist;
8999                 unsigned int i;
9000                 u32 mc_filter[4] = { 0, };
9001                 u32 regidx;
9002                 u32 bit;
9003                 u32 crc;
9004
9005                 for (i = 0, mclist = dev->mc_list; mclist && i < dev->mc_count;
9006                      i++, mclist = mclist->next) {
9007
9008                         crc = calc_crc (mclist->dmi_addr, ETH_ALEN);
9009                         bit = ~crc & 0x7f;
9010                         regidx = (bit & 0x60) >> 5;
9011                         bit &= 0x1f;
9012                         mc_filter[regidx] |= (1 << bit);
9013                 }
9014
9015                 tw32(MAC_HASH_REG_0, mc_filter[0]);
9016                 tw32(MAC_HASH_REG_1, mc_filter[1]);
9017                 tw32(MAC_HASH_REG_2, mc_filter[2]);
9018                 tw32(MAC_HASH_REG_3, mc_filter[3]);
9019         }
9020
9021         if (rx_mode != tp->rx_mode) {
9022                 tp->rx_mode = rx_mode;
9023                 tw32_f(MAC_RX_MODE, rx_mode);
9024                 udelay(10);
9025         }
9026 }
9027
9028 static void tg3_set_rx_mode(struct net_device *dev)
9029 {
9030         struct tg3 *tp = netdev_priv(dev);
9031
9032         if (!netif_running(dev))
9033                 return;
9034
9035         tg3_full_lock(tp, 0);
9036         __tg3_set_rx_mode(dev);
9037         tg3_full_unlock(tp);
9038 }
9039
9040 #define TG3_REGDUMP_LEN         (32 * 1024)
9041
9042 static int tg3_get_regs_len(struct net_device *dev)
9043 {
9044         return TG3_REGDUMP_LEN;
9045 }
9046
9047 static void tg3_get_regs(struct net_device *dev,
9048                 struct ethtool_regs *regs, void *_p)
9049 {
9050         u32 *p = _p;
9051         struct tg3 *tp = netdev_priv(dev);
9052         u8 *orig_p = _p;
9053         int i;
9054
9055         regs->version = 0;
9056
9057         memset(p, 0, TG3_REGDUMP_LEN);
9058
9059         if (tp->link_config.phy_is_low_power)
9060                 return;
9061
9062         tg3_full_lock(tp, 0);
9063
9064 #define __GET_REG32(reg)        (*(p)++ = tr32(reg))
9065 #define GET_REG32_LOOP(base,len)                \
9066 do {    p = (u32 *)(orig_p + (base));           \
9067         for (i = 0; i < len; i += 4)            \
9068                 __GET_REG32((base) + i);        \
9069 } while (0)
9070 #define GET_REG32_1(reg)                        \
9071 do {    p = (u32 *)(orig_p + (reg));            \
9072         __GET_REG32((reg));                     \
9073 } while (0)
9074
9075         GET_REG32_LOOP(TG3PCI_VENDOR, 0xb0);
9076         GET_REG32_LOOP(MAILBOX_INTERRUPT_0, 0x200);
9077         GET_REG32_LOOP(MAC_MODE, 0x4f0);
9078         GET_REG32_LOOP(SNDDATAI_MODE, 0xe0);
9079         GET_REG32_1(SNDDATAC_MODE);
9080         GET_REG32_LOOP(SNDBDS_MODE, 0x80);
9081         GET_REG32_LOOP(SNDBDI_MODE, 0x48);
9082         GET_REG32_1(SNDBDC_MODE);
9083         GET_REG32_LOOP(RCVLPC_MODE, 0x20);
9084         GET_REG32_LOOP(RCVLPC_SELLST_BASE, 0x15c);
9085         GET_REG32_LOOP(RCVDBDI_MODE, 0x0c);
9086         GET_REG32_LOOP(RCVDBDI_JUMBO_BD, 0x3c);
9087         GET_REG32_LOOP(RCVDBDI_BD_PROD_IDX_0, 0x44);
9088         GET_REG32_1(RCVDCC_MODE);
9089         GET_REG32_LOOP(RCVBDI_MODE, 0x20);
9090         GET_REG32_LOOP(RCVCC_MODE, 0x14);
9091         GET_REG32_LOOP(RCVLSC_MODE, 0x08);
9092         GET_REG32_1(MBFREE_MODE);
9093         GET_REG32_LOOP(HOSTCC_MODE, 0x100);
9094         GET_REG32_LOOP(MEMARB_MODE, 0x10);
9095         GET_REG32_LOOP(BUFMGR_MODE, 0x58);
9096         GET_REG32_LOOP(RDMAC_MODE, 0x08);
9097         GET_REG32_LOOP(WDMAC_MODE, 0x08);
9098         GET_REG32_1(RX_CPU_MODE);
9099         GET_REG32_1(RX_CPU_STATE);
9100         GET_REG32_1(RX_CPU_PGMCTR);
9101         GET_REG32_1(RX_CPU_HWBKPT);
9102         GET_REG32_1(TX_CPU_MODE);
9103         GET_REG32_1(TX_CPU_STATE);
9104         GET_REG32_1(TX_CPU_PGMCTR);
9105         GET_REG32_LOOP(GRCMBOX_INTERRUPT_0, 0x110);
9106         GET_REG32_LOOP(FTQ_RESET, 0x120);
9107         GET_REG32_LOOP(MSGINT_MODE, 0x0c);
9108         GET_REG32_1(DMAC_MODE);
9109         GET_REG32_LOOP(GRC_MODE, 0x4c);
9110         if (tp->tg3_flags & TG3_FLAG_NVRAM)
9111                 GET_REG32_LOOP(NVRAM_CMD, 0x24);
9112
9113 #undef __GET_REG32
9114 #undef GET_REG32_LOOP
9115 #undef GET_REG32_1
9116
9117         tg3_full_unlock(tp);
9118 }
9119
9120 static int tg3_get_eeprom_len(struct net_device *dev)
9121 {
9122         struct tg3 *tp = netdev_priv(dev);
9123
9124         return tp->nvram_size;
9125 }
9126
9127 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
9128 {
9129         struct tg3 *tp = netdev_priv(dev);
9130         int ret;
9131         u8  *pd;
9132         u32 i, offset, len, b_offset, b_count;
9133         __be32 val;
9134
9135         if (tp->tg3_flags3 & TG3_FLG3_NO_NVRAM)
9136                 return -EINVAL;
9137
9138         if (tp->link_config.phy_is_low_power)
9139                 return -EAGAIN;
9140
9141         offset = eeprom->offset;
9142         len = eeprom->len;
9143         eeprom->len = 0;
9144
9145         eeprom->magic = TG3_EEPROM_MAGIC;
9146
9147         if (offset & 3) {
9148                 /* adjustments to start on required 4 byte boundary */
9149                 b_offset = offset & 3;
9150                 b_count = 4 - b_offset;
9151                 if (b_count > len) {
9152                         /* i.e. offset=1 len=2 */
9153                         b_count = len;
9154                 }
9155                 ret = tg3_nvram_read_be32(tp, offset-b_offset, &val);
9156                 if (ret)
9157                         return ret;
9158                 memcpy(data, ((char*)&val) + b_offset, b_count);
9159                 len -= b_count;
9160                 offset += b_count;
9161                 eeprom->len += b_count;
9162         }
9163
9164         /* read bytes upto the last 4 byte boundary */
9165         pd = &data[eeprom->len];
9166         for (i = 0; i < (len - (len & 3)); i += 4) {
9167                 ret = tg3_nvram_read_be32(tp, offset + i, &val);
9168                 if (ret) {
9169                         eeprom->len += i;
9170                         return ret;
9171                 }
9172                 memcpy(pd + i, &val, 4);
9173         }
9174         eeprom->len += i;
9175
9176         if (len & 3) {
9177                 /* read last bytes not ending on 4 byte boundary */
9178                 pd = &data[eeprom->len];
9179                 b_count = len & 3;
9180                 b_offset = offset + len - b_count;
9181                 ret = tg3_nvram_read_be32(tp, b_offset, &val);
9182                 if (ret)
9183                         return ret;
9184                 memcpy(pd, &val, b_count);
9185                 eeprom->len += b_count;
9186         }
9187         return 0;
9188 }
9189
9190 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf);
9191
9192 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
9193 {
9194         struct tg3 *tp = netdev_priv(dev);
9195         int ret;
9196         u32 offset, len, b_offset, odd_len;
9197         u8 *buf;
9198         __be32 start, end;
9199
9200         if (tp->link_config.phy_is_low_power)
9201                 return -EAGAIN;
9202
9203         if ((tp->tg3_flags3 & TG3_FLG3_NO_NVRAM) ||
9204             eeprom->magic != TG3_EEPROM_MAGIC)
9205                 return -EINVAL;
9206
9207         offset = eeprom->offset;
9208         len = eeprom->len;
9209
9210         if ((b_offset = (offset & 3))) {
9211                 /* adjustments to start on required 4 byte boundary */
9212                 ret = tg3_nvram_read_be32(tp, offset-b_offset, &start);
9213                 if (ret)
9214                         return ret;
9215                 len += b_offset;
9216                 offset &= ~3;
9217                 if (len < 4)
9218                         len = 4;
9219         }
9220
9221         odd_len = 0;
9222         if (len & 3) {
9223                 /* adjustments to end on required 4 byte boundary */
9224                 odd_len = 1;
9225                 len = (len + 3) & ~3;
9226                 ret = tg3_nvram_read_be32(tp, offset+len-4, &end);
9227                 if (ret)
9228                         return ret;
9229         }
9230
9231         buf = data;
9232         if (b_offset || odd_len) {
9233                 buf = kmalloc(len, GFP_KERNEL);
9234                 if (!buf)
9235                         return -ENOMEM;
9236                 if (b_offset)
9237                         memcpy(buf, &start, 4);
9238                 if (odd_len)
9239                         memcpy(buf+len-4, &end, 4);
9240                 memcpy(buf + b_offset, data, eeprom->len);
9241         }
9242
9243         ret = tg3_nvram_write_block(tp, offset, len, buf);
9244
9245         if (buf != data)
9246                 kfree(buf);
9247
9248         return ret;
9249 }
9250
9251 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
9252 {
9253         struct tg3 *tp = netdev_priv(dev);
9254
9255         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
9256                 if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
9257                         return -EAGAIN;
9258                 return phy_ethtool_gset(tp->mdio_bus->phy_map[PHY_ADDR], cmd);
9259         }
9260
9261         cmd->supported = (SUPPORTED_Autoneg);
9262
9263         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
9264                 cmd->supported |= (SUPPORTED_1000baseT_Half |
9265                                    SUPPORTED_1000baseT_Full);
9266
9267         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)) {
9268                 cmd->supported |= (SUPPORTED_100baseT_Half |
9269                                   SUPPORTED_100baseT_Full |
9270                                   SUPPORTED_10baseT_Half |
9271                                   SUPPORTED_10baseT_Full |
9272                                   SUPPORTED_TP);
9273                 cmd->port = PORT_TP;
9274         } else {
9275                 cmd->supported |= SUPPORTED_FIBRE;
9276                 cmd->port = PORT_FIBRE;
9277         }
9278
9279         cmd->advertising = tp->link_config.advertising;
9280         if (netif_running(dev)) {
9281                 cmd->speed = tp->link_config.active_speed;
9282                 cmd->duplex = tp->link_config.active_duplex;
9283         }
9284         cmd->phy_address = tp->phy_addr;
9285         cmd->transceiver = XCVR_INTERNAL;
9286         cmd->autoneg = tp->link_config.autoneg;
9287         cmd->maxtxpkt = 0;
9288         cmd->maxrxpkt = 0;
9289         return 0;
9290 }
9291
9292 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
9293 {
9294         struct tg3 *tp = netdev_priv(dev);
9295
9296         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
9297                 if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
9298                         return -EAGAIN;
9299                 return phy_ethtool_sset(tp->mdio_bus->phy_map[PHY_ADDR], cmd);
9300         }
9301
9302         if (cmd->autoneg != AUTONEG_ENABLE &&
9303             cmd->autoneg != AUTONEG_DISABLE)
9304                 return -EINVAL;
9305
9306         if (cmd->autoneg == AUTONEG_DISABLE &&
9307             cmd->duplex != DUPLEX_FULL &&
9308             cmd->duplex != DUPLEX_HALF)
9309                 return -EINVAL;
9310
9311         if (cmd->autoneg == AUTONEG_ENABLE) {
9312                 u32 mask = ADVERTISED_Autoneg |
9313                            ADVERTISED_Pause |
9314                            ADVERTISED_Asym_Pause;
9315
9316                 if (!(tp->tg3_flags2 & TG3_FLAG_10_100_ONLY))
9317                         mask |= ADVERTISED_1000baseT_Half |
9318                                 ADVERTISED_1000baseT_Full;
9319
9320                 if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
9321                         mask |= ADVERTISED_100baseT_Half |
9322                                 ADVERTISED_100baseT_Full |
9323                                 ADVERTISED_10baseT_Half |
9324                                 ADVERTISED_10baseT_Full |
9325                                 ADVERTISED_TP;
9326                 else
9327                         mask |= ADVERTISED_FIBRE;
9328
9329                 if (cmd->advertising & ~mask)
9330                         return -EINVAL;
9331
9332                 mask &= (ADVERTISED_1000baseT_Half |
9333                          ADVERTISED_1000baseT_Full |
9334                          ADVERTISED_100baseT_Half |
9335                          ADVERTISED_100baseT_Full |
9336                          ADVERTISED_10baseT_Half |
9337                          ADVERTISED_10baseT_Full);
9338
9339                 cmd->advertising &= mask;
9340         } else {
9341                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) {
9342                         if (cmd->speed != SPEED_1000)
9343                                 return -EINVAL;
9344
9345                         if (cmd->duplex != DUPLEX_FULL)
9346                                 return -EINVAL;
9347                 } else {
9348                         if (cmd->speed != SPEED_100 &&
9349                             cmd->speed != SPEED_10)
9350                                 return -EINVAL;
9351                 }
9352         }
9353
9354         tg3_full_lock(tp, 0);
9355
9356         tp->link_config.autoneg = cmd->autoneg;
9357         if (cmd->autoneg == AUTONEG_ENABLE) {
9358                 tp->link_config.advertising = (cmd->advertising |
9359                                               ADVERTISED_Autoneg);
9360                 tp->link_config.speed = SPEED_INVALID;
9361                 tp->link_config.duplex = DUPLEX_INVALID;
9362         } else {
9363                 tp->link_config.advertising = 0;
9364                 tp->link_config.speed = cmd->speed;
9365                 tp->link_config.duplex = cmd->duplex;
9366         }
9367
9368         tp->link_config.orig_speed = tp->link_config.speed;
9369         tp->link_config.orig_duplex = tp->link_config.duplex;
9370         tp->link_config.orig_autoneg = tp->link_config.autoneg;
9371
9372         if (netif_running(dev))
9373                 tg3_setup_phy(tp, 1);
9374
9375         tg3_full_unlock(tp);
9376
9377         return 0;
9378 }
9379
9380 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
9381 {
9382         struct tg3 *tp = netdev_priv(dev);
9383
9384         strcpy(info->driver, DRV_MODULE_NAME);
9385         strcpy(info->version, DRV_MODULE_VERSION);
9386         strcpy(info->fw_version, tp->fw_ver);
9387         strcpy(info->bus_info, pci_name(tp->pdev));
9388 }
9389
9390 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
9391 {
9392         struct tg3 *tp = netdev_priv(dev);
9393
9394         if ((tp->tg3_flags & TG3_FLAG_WOL_CAP) &&
9395             device_can_wakeup(&tp->pdev->dev))
9396                 wol->supported = WAKE_MAGIC;
9397         else
9398                 wol->supported = 0;
9399         wol->wolopts = 0;
9400         if ((tp->tg3_flags & TG3_FLAG_WOL_ENABLE) &&
9401             device_can_wakeup(&tp->pdev->dev))
9402                 wol->wolopts = WAKE_MAGIC;
9403         memset(&wol->sopass, 0, sizeof(wol->sopass));
9404 }
9405
9406 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
9407 {
9408         struct tg3 *tp = netdev_priv(dev);
9409         struct device *dp = &tp->pdev->dev;
9410
9411         if (wol->wolopts & ~WAKE_MAGIC)
9412                 return -EINVAL;
9413         if ((wol->wolopts & WAKE_MAGIC) &&
9414             !((tp->tg3_flags & TG3_FLAG_WOL_CAP) && device_can_wakeup(dp)))
9415                 return -EINVAL;
9416
9417         spin_lock_bh(&tp->lock);
9418         if (wol->wolopts & WAKE_MAGIC) {
9419                 tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
9420                 device_set_wakeup_enable(dp, true);
9421         } else {
9422                 tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE;
9423                 device_set_wakeup_enable(dp, false);
9424         }
9425         spin_unlock_bh(&tp->lock);
9426
9427         return 0;
9428 }
9429
9430 static u32 tg3_get_msglevel(struct net_device *dev)
9431 {
9432         struct tg3 *tp = netdev_priv(dev);
9433         return tp->msg_enable;
9434 }
9435
9436 static void tg3_set_msglevel(struct net_device *dev, u32 value)
9437 {
9438         struct tg3 *tp = netdev_priv(dev);
9439         tp->msg_enable = value;
9440 }
9441
9442 static int tg3_set_tso(struct net_device *dev, u32 value)
9443 {
9444         struct tg3 *tp = netdev_priv(dev);
9445
9446         if (!(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE)) {
9447                 if (value)
9448                         return -EINVAL;
9449                 return 0;
9450         }
9451         if ((dev->features & NETIF_F_IPV6_CSUM) &&
9452             (tp->tg3_flags2 & TG3_FLG2_HW_TSO_2)) {
9453                 if (value) {
9454                         dev->features |= NETIF_F_TSO6;
9455                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
9456                             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
9457                              GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) ||
9458                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
9459                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
9460                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
9461                                 dev->features |= NETIF_F_TSO_ECN;
9462                 } else
9463                         dev->features &= ~(NETIF_F_TSO6 | NETIF_F_TSO_ECN);
9464         }
9465         return ethtool_op_set_tso(dev, value);
9466 }
9467
9468 static int tg3_nway_reset(struct net_device *dev)
9469 {
9470         struct tg3 *tp = netdev_priv(dev);
9471         int r;
9472
9473         if (!netif_running(dev))
9474                 return -EAGAIN;
9475
9476         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
9477                 return -EINVAL;
9478
9479         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
9480                 if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
9481                         return -EAGAIN;
9482                 r = phy_start_aneg(tp->mdio_bus->phy_map[PHY_ADDR]);
9483         } else {
9484                 u32 bmcr;
9485
9486                 spin_lock_bh(&tp->lock);
9487                 r = -EINVAL;
9488                 tg3_readphy(tp, MII_BMCR, &bmcr);
9489                 if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
9490                     ((bmcr & BMCR_ANENABLE) ||
9491                      (tp->tg3_flags2 & TG3_FLG2_PARALLEL_DETECT))) {
9492                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
9493                                                    BMCR_ANENABLE);
9494                         r = 0;
9495                 }
9496                 spin_unlock_bh(&tp->lock);
9497         }
9498
9499         return r;
9500 }
9501
9502 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
9503 {
9504         struct tg3 *tp = netdev_priv(dev);
9505
9506         ering->rx_max_pending = TG3_RX_RING_SIZE - 1;
9507         ering->rx_mini_max_pending = 0;
9508         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
9509                 ering->rx_jumbo_max_pending = TG3_RX_JUMBO_RING_SIZE - 1;
9510         else
9511                 ering->rx_jumbo_max_pending = 0;
9512
9513         ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
9514
9515         ering->rx_pending = tp->rx_pending;
9516         ering->rx_mini_pending = 0;
9517         if (tp->tg3_flags & TG3_FLAG_JUMBO_RING_ENABLE)
9518                 ering->rx_jumbo_pending = tp->rx_jumbo_pending;
9519         else
9520                 ering->rx_jumbo_pending = 0;
9521
9522         ering->tx_pending = tp->napi[0].tx_pending;
9523 }
9524
9525 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
9526 {
9527         struct tg3 *tp = netdev_priv(dev);
9528         int i, irq_sync = 0, err = 0;
9529
9530         if ((ering->rx_pending > TG3_RX_RING_SIZE - 1) ||
9531             (ering->rx_jumbo_pending > TG3_RX_JUMBO_RING_SIZE - 1) ||
9532             (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
9533             (ering->tx_pending <= MAX_SKB_FRAGS) ||
9534             ((tp->tg3_flags2 & TG3_FLG2_TSO_BUG) &&
9535              (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
9536                 return -EINVAL;
9537
9538         if (netif_running(dev)) {
9539                 tg3_phy_stop(tp);
9540                 tg3_netif_stop(tp);
9541                 irq_sync = 1;
9542         }
9543
9544         tg3_full_lock(tp, irq_sync);
9545
9546         tp->rx_pending = ering->rx_pending;
9547
9548         if ((tp->tg3_flags2 & TG3_FLG2_MAX_RXPEND_64) &&
9549             tp->rx_pending > 63)
9550                 tp->rx_pending = 63;
9551         tp->rx_jumbo_pending = ering->rx_jumbo_pending;
9552
9553         for (i = 0; i < TG3_IRQ_MAX_VECS; i++)
9554                 tp->napi[i].tx_pending = ering->tx_pending;
9555
9556         if (netif_running(dev)) {
9557                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9558                 err = tg3_restart_hw(tp, 1);
9559                 if (!err)
9560                         tg3_netif_start(tp);
9561         }
9562
9563         tg3_full_unlock(tp);
9564
9565         if (irq_sync && !err)
9566                 tg3_phy_start(tp);
9567
9568         return err;
9569 }
9570
9571 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
9572 {
9573         struct tg3 *tp = netdev_priv(dev);
9574
9575         epause->autoneg = (tp->tg3_flags & TG3_FLAG_PAUSE_AUTONEG) != 0;
9576
9577         if (tp->link_config.active_flowctrl & FLOW_CTRL_RX)
9578                 epause->rx_pause = 1;
9579         else
9580                 epause->rx_pause = 0;
9581
9582         if (tp->link_config.active_flowctrl & FLOW_CTRL_TX)
9583                 epause->tx_pause = 1;
9584         else
9585                 epause->tx_pause = 0;
9586 }
9587
9588 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
9589 {
9590         struct tg3 *tp = netdev_priv(dev);
9591         int err = 0;
9592
9593         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
9594                 if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
9595                         return -EAGAIN;
9596
9597                 if (epause->autoneg) {
9598                         u32 newadv;
9599                         struct phy_device *phydev;
9600
9601                         phydev = tp->mdio_bus->phy_map[PHY_ADDR];
9602
9603                         if (epause->rx_pause) {
9604                                 if (epause->tx_pause)
9605                                         newadv = ADVERTISED_Pause;
9606                                 else
9607                                         newadv = ADVERTISED_Pause |
9608                                                  ADVERTISED_Asym_Pause;
9609                         } else if (epause->tx_pause) {
9610                                 newadv = ADVERTISED_Asym_Pause;
9611                         } else
9612                                 newadv = 0;
9613
9614                         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) {
9615                                 u32 oldadv = phydev->advertising &
9616                                              (ADVERTISED_Pause |
9617                                               ADVERTISED_Asym_Pause);
9618                                 if (oldadv != newadv) {
9619                                         phydev->advertising &=
9620                                                 ~(ADVERTISED_Pause |
9621                                                   ADVERTISED_Asym_Pause);
9622                                         phydev->advertising |= newadv;
9623                                         err = phy_start_aneg(phydev);
9624                                 }
9625                         } else {
9626                                 tp->link_config.advertising &=
9627                                                 ~(ADVERTISED_Pause |
9628                                                   ADVERTISED_Asym_Pause);
9629                                 tp->link_config.advertising |= newadv;
9630                         }
9631                 } else {
9632                         if (epause->rx_pause)
9633                                 tp->link_config.flowctrl |= FLOW_CTRL_RX;
9634                         else
9635                                 tp->link_config.flowctrl &= ~FLOW_CTRL_RX;
9636
9637                         if (epause->tx_pause)
9638                                 tp->link_config.flowctrl |= FLOW_CTRL_TX;
9639                         else
9640                                 tp->link_config.flowctrl &= ~FLOW_CTRL_TX;
9641
9642                         if (netif_running(dev))
9643                                 tg3_setup_flow_control(tp, 0, 0);
9644                 }
9645         } else {
9646                 int irq_sync = 0;
9647
9648                 if (netif_running(dev)) {
9649                         tg3_netif_stop(tp);
9650                         irq_sync = 1;
9651                 }
9652
9653                 tg3_full_lock(tp, irq_sync);
9654
9655                 if (epause->autoneg)
9656                         tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
9657                 else
9658                         tp->tg3_flags &= ~TG3_FLAG_PAUSE_AUTONEG;
9659                 if (epause->rx_pause)
9660                         tp->link_config.flowctrl |= FLOW_CTRL_RX;
9661                 else
9662                         tp->link_config.flowctrl &= ~FLOW_CTRL_RX;
9663                 if (epause->tx_pause)
9664                         tp->link_config.flowctrl |= FLOW_CTRL_TX;
9665                 else
9666                         tp->link_config.flowctrl &= ~FLOW_CTRL_TX;
9667
9668                 if (netif_running(dev)) {
9669                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9670                         err = tg3_restart_hw(tp, 1);
9671                         if (!err)
9672                                 tg3_netif_start(tp);
9673                 }
9674
9675                 tg3_full_unlock(tp);
9676         }
9677
9678         return err;
9679 }
9680
9681 static u32 tg3_get_rx_csum(struct net_device *dev)
9682 {
9683         struct tg3 *tp = netdev_priv(dev);
9684         return (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0;
9685 }
9686
9687 static int tg3_set_rx_csum(struct net_device *dev, u32 data)
9688 {
9689         struct tg3 *tp = netdev_priv(dev);
9690
9691         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
9692                 if (data != 0)
9693                         return -EINVAL;
9694                 return 0;
9695         }
9696
9697         spin_lock_bh(&tp->lock);
9698         if (data)
9699                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
9700         else
9701                 tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS;
9702         spin_unlock_bh(&tp->lock);
9703
9704         return 0;
9705 }
9706
9707 static int tg3_set_tx_csum(struct net_device *dev, u32 data)
9708 {
9709         struct tg3 *tp = netdev_priv(dev);
9710
9711         if (tp->tg3_flags & TG3_FLAG_BROKEN_CHECKSUMS) {
9712                 if (data != 0)
9713                         return -EINVAL;
9714                 return 0;
9715         }
9716
9717         if (tp->tg3_flags3 & TG3_FLG3_5755_PLUS)
9718                 ethtool_op_set_tx_ipv6_csum(dev, data);
9719         else
9720                 ethtool_op_set_tx_csum(dev, data);
9721
9722         return 0;
9723 }
9724
9725 static int tg3_get_sset_count (struct net_device *dev, int sset)
9726 {
9727         switch (sset) {
9728         case ETH_SS_TEST:
9729                 return TG3_NUM_TEST;
9730         case ETH_SS_STATS:
9731                 return TG3_NUM_STATS;
9732         default:
9733                 return -EOPNOTSUPP;
9734         }
9735 }
9736
9737 static void tg3_get_strings (struct net_device *dev, u32 stringset, u8 *buf)
9738 {
9739         switch (stringset) {
9740         case ETH_SS_STATS:
9741                 memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
9742                 break;
9743         case ETH_SS_TEST:
9744                 memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
9745                 break;
9746         default:
9747                 WARN_ON(1);     /* we need a WARN() */
9748                 break;
9749         }
9750 }
9751
9752 static int tg3_phys_id(struct net_device *dev, u32 data)
9753 {
9754         struct tg3 *tp = netdev_priv(dev);
9755         int i;
9756
9757         if (!netif_running(tp->dev))
9758                 return -EAGAIN;
9759
9760         if (data == 0)
9761                 data = UINT_MAX / 2;
9762
9763         for (i = 0; i < (data * 2); i++) {
9764                 if ((i % 2) == 0)
9765                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
9766                                            LED_CTRL_1000MBPS_ON |
9767                                            LED_CTRL_100MBPS_ON |
9768                                            LED_CTRL_10MBPS_ON |
9769                                            LED_CTRL_TRAFFIC_OVERRIDE |
9770                                            LED_CTRL_TRAFFIC_BLINK |
9771                                            LED_CTRL_TRAFFIC_LED);
9772
9773                 else
9774                         tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
9775                                            LED_CTRL_TRAFFIC_OVERRIDE);
9776
9777                 if (msleep_interruptible(500))
9778                         break;
9779         }
9780         tw32(MAC_LED_CTRL, tp->led_ctrl);
9781         return 0;
9782 }
9783
9784 static void tg3_get_ethtool_stats (struct net_device *dev,
9785                                    struct ethtool_stats *estats, u64 *tmp_stats)
9786 {
9787         struct tg3 *tp = netdev_priv(dev);
9788         memcpy(tmp_stats, tg3_get_estats(tp), sizeof(tp->estats));
9789 }
9790
9791 #define NVRAM_TEST_SIZE 0x100
9792 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE   0x14
9793 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE   0x18
9794 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE   0x1c
9795 #define NVRAM_SELFBOOT_HW_SIZE 0x20
9796 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
9797
9798 static int tg3_test_nvram(struct tg3 *tp)
9799 {
9800         u32 csum, magic;
9801         __be32 *buf;
9802         int i, j, k, err = 0, size;
9803
9804         if (tp->tg3_flags3 & TG3_FLG3_NO_NVRAM)
9805                 return 0;
9806
9807         if (tg3_nvram_read(tp, 0, &magic) != 0)
9808                 return -EIO;
9809
9810         if (magic == TG3_EEPROM_MAGIC)
9811                 size = NVRAM_TEST_SIZE;
9812         else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
9813                 if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
9814                     TG3_EEPROM_SB_FORMAT_1) {
9815                         switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
9816                         case TG3_EEPROM_SB_REVISION_0:
9817                                 size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
9818                                 break;
9819                         case TG3_EEPROM_SB_REVISION_2:
9820                                 size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
9821                                 break;
9822                         case TG3_EEPROM_SB_REVISION_3:
9823                                 size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
9824                                 break;
9825                         default:
9826                                 return 0;
9827                         }
9828                 } else
9829                         return 0;
9830         } else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
9831                 size = NVRAM_SELFBOOT_HW_SIZE;
9832         else
9833                 return -EIO;
9834
9835         buf = kmalloc(size, GFP_KERNEL);
9836         if (buf == NULL)
9837                 return -ENOMEM;
9838
9839         err = -EIO;
9840         for (i = 0, j = 0; i < size; i += 4, j++) {
9841                 err = tg3_nvram_read_be32(tp, i, &buf[j]);
9842                 if (err)
9843                         break;
9844         }
9845         if (i < size)
9846                 goto out;
9847
9848         /* Selfboot format */
9849         magic = be32_to_cpu(buf[0]);
9850         if ((magic & TG3_EEPROM_MAGIC_FW_MSK) ==
9851             TG3_EEPROM_MAGIC_FW) {
9852                 u8 *buf8 = (u8 *) buf, csum8 = 0;
9853
9854                 if ((magic & TG3_EEPROM_SB_REVISION_MASK) ==
9855                     TG3_EEPROM_SB_REVISION_2) {
9856                         /* For rev 2, the csum doesn't include the MBA. */
9857                         for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
9858                                 csum8 += buf8[i];
9859                         for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
9860                                 csum8 += buf8[i];
9861                 } else {
9862                         for (i = 0; i < size; i++)
9863                                 csum8 += buf8[i];
9864                 }
9865
9866                 if (csum8 == 0) {
9867                         err = 0;
9868                         goto out;
9869                 }
9870
9871                 err = -EIO;
9872                 goto out;
9873         }
9874
9875         if ((magic & TG3_EEPROM_MAGIC_HW_MSK) ==
9876             TG3_EEPROM_MAGIC_HW) {
9877                 u8 data[NVRAM_SELFBOOT_DATA_SIZE];
9878                 u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
9879                 u8 *buf8 = (u8 *) buf;
9880
9881                 /* Separate the parity bits and the data bytes.  */
9882                 for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
9883                         if ((i == 0) || (i == 8)) {
9884                                 int l;
9885                                 u8 msk;
9886
9887                                 for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
9888                                         parity[k++] = buf8[i] & msk;
9889                                 i++;
9890                         }
9891                         else if (i == 16) {
9892                                 int l;
9893                                 u8 msk;
9894
9895                                 for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
9896                                         parity[k++] = buf8[i] & msk;
9897                                 i++;
9898
9899                                 for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
9900                                         parity[k++] = buf8[i] & msk;
9901                                 i++;
9902                         }
9903                         data[j++] = buf8[i];
9904                 }
9905
9906                 err = -EIO;
9907                 for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
9908                         u8 hw8 = hweight8(data[i]);
9909
9910                         if ((hw8 & 0x1) && parity[i])
9911                                 goto out;
9912                         else if (!(hw8 & 0x1) && !parity[i])
9913                                 goto out;
9914                 }
9915                 err = 0;
9916                 goto out;
9917         }
9918
9919         /* Bootstrap checksum at offset 0x10 */
9920         csum = calc_crc((unsigned char *) buf, 0x10);
9921         if (csum != be32_to_cpu(buf[0x10/4]))
9922                 goto out;
9923
9924         /* Manufacturing block starts at offset 0x74, checksum at 0xfc */
9925         csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
9926         if (csum != be32_to_cpu(buf[0xfc/4]))
9927                 goto out;
9928
9929         err = 0;
9930
9931 out:
9932         kfree(buf);
9933         return err;
9934 }
9935
9936 #define TG3_SERDES_TIMEOUT_SEC  2
9937 #define TG3_COPPER_TIMEOUT_SEC  6
9938
9939 static int tg3_test_link(struct tg3 *tp)
9940 {
9941         int i, max;
9942
9943         if (!netif_running(tp->dev))
9944                 return -ENODEV;
9945
9946         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
9947                 max = TG3_SERDES_TIMEOUT_SEC;
9948         else
9949                 max = TG3_COPPER_TIMEOUT_SEC;
9950
9951         for (i = 0; i < max; i++) {
9952                 if (netif_carrier_ok(tp->dev))
9953                         return 0;
9954
9955                 if (msleep_interruptible(1000))
9956                         break;
9957         }
9958
9959         return -EIO;
9960 }
9961
9962 /* Only test the commonly used registers */
9963 static int tg3_test_registers(struct tg3 *tp)
9964 {
9965         int i, is_5705, is_5750;
9966         u32 offset, read_mask, write_mask, val, save_val, read_val;
9967         static struct {
9968                 u16 offset;
9969                 u16 flags;
9970 #define TG3_FL_5705     0x1
9971 #define TG3_FL_NOT_5705 0x2
9972 #define TG3_FL_NOT_5788 0x4
9973 #define TG3_FL_NOT_5750 0x8
9974                 u32 read_mask;
9975                 u32 write_mask;
9976         } reg_tbl[] = {
9977                 /* MAC Control Registers */
9978                 { MAC_MODE, TG3_FL_NOT_5705,
9979                         0x00000000, 0x00ef6f8c },
9980                 { MAC_MODE, TG3_FL_5705,
9981                         0x00000000, 0x01ef6b8c },
9982                 { MAC_STATUS, TG3_FL_NOT_5705,
9983                         0x03800107, 0x00000000 },
9984                 { MAC_STATUS, TG3_FL_5705,
9985                         0x03800100, 0x00000000 },
9986                 { MAC_ADDR_0_HIGH, 0x0000,
9987                         0x00000000, 0x0000ffff },
9988                 { MAC_ADDR_0_LOW, 0x0000,
9989                         0x00000000, 0xffffffff },
9990                 { MAC_RX_MTU_SIZE, 0x0000,
9991                         0x00000000, 0x0000ffff },
9992                 { MAC_TX_MODE, 0x0000,
9993                         0x00000000, 0x00000070 },
9994                 { MAC_TX_LENGTHS, 0x0000,
9995                         0x00000000, 0x00003fff },
9996                 { MAC_RX_MODE, TG3_FL_NOT_5705,
9997                         0x00000000, 0x000007fc },
9998                 { MAC_RX_MODE, TG3_FL_5705,
9999                         0x00000000, 0x000007dc },
10000                 { MAC_HASH_REG_0, 0x0000,
10001                         0x00000000, 0xffffffff },
10002                 { MAC_HASH_REG_1, 0x0000,
10003                         0x00000000, 0xffffffff },
10004                 { MAC_HASH_REG_2, 0x0000,
10005                         0x00000000, 0xffffffff },
10006                 { MAC_HASH_REG_3, 0x0000,
10007                         0x00000000, 0xffffffff },
10008
10009                 /* Receive Data and Receive BD Initiator Control Registers. */
10010                 { RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
10011                         0x00000000, 0xffffffff },
10012                 { RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
10013                         0x00000000, 0xffffffff },
10014                 { RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
10015                         0x00000000, 0x00000003 },
10016                 { RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
10017                         0x00000000, 0xffffffff },
10018                 { RCVDBDI_STD_BD+0, 0x0000,
10019                         0x00000000, 0xffffffff },
10020                 { RCVDBDI_STD_BD+4, 0x0000,
10021                         0x00000000, 0xffffffff },
10022                 { RCVDBDI_STD_BD+8, 0x0000,
10023                         0x00000000, 0xffff0002 },
10024                 { RCVDBDI_STD_BD+0xc, 0x0000,
10025                         0x00000000, 0xffffffff },
10026
10027                 /* Receive BD Initiator Control Registers. */
10028                 { RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
10029                         0x00000000, 0xffffffff },
10030                 { RCVBDI_STD_THRESH, TG3_FL_5705,
10031                         0x00000000, 0x000003ff },
10032                 { RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
10033                         0x00000000, 0xffffffff },
10034
10035                 /* Host Coalescing Control Registers. */
10036                 { HOSTCC_MODE, TG3_FL_NOT_5705,
10037                         0x00000000, 0x00000004 },
10038                 { HOSTCC_MODE, TG3_FL_5705,
10039                         0x00000000, 0x000000f6 },
10040                 { HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
10041                         0x00000000, 0xffffffff },
10042                 { HOSTCC_RXCOL_TICKS, TG3_FL_5705,
10043                         0x00000000, 0x000003ff },
10044                 { HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
10045                         0x00000000, 0xffffffff },
10046                 { HOSTCC_TXCOL_TICKS, TG3_FL_5705,
10047                         0x00000000, 0x000003ff },
10048                 { HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
10049                         0x00000000, 0xffffffff },
10050                 { HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
10051                         0x00000000, 0x000000ff },
10052                 { HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
10053                         0x00000000, 0xffffffff },
10054                 { HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
10055                         0x00000000, 0x000000ff },
10056                 { HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
10057                         0x00000000, 0xffffffff },
10058                 { HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
10059                         0x00000000, 0xffffffff },
10060                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
10061                         0x00000000, 0xffffffff },
10062                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
10063                         0x00000000, 0x000000ff },
10064                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
10065                         0x00000000, 0xffffffff },
10066                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
10067                         0x00000000, 0x000000ff },
10068                 { HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
10069                         0x00000000, 0xffffffff },
10070                 { HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
10071                         0x00000000, 0xffffffff },
10072                 { HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
10073                         0x00000000, 0xffffffff },
10074                 { HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
10075                         0x00000000, 0xffffffff },
10076                 { HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
10077                         0x00000000, 0xffffffff },
10078                 { HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
10079                         0xffffffff, 0x00000000 },
10080                 { HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
10081                         0xffffffff, 0x00000000 },
10082
10083                 /* Buffer Manager Control Registers. */
10084                 { BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
10085                         0x00000000, 0x007fff80 },
10086                 { BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
10087                         0x00000000, 0x007fffff },
10088                 { BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
10089                         0x00000000, 0x0000003f },
10090                 { BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
10091                         0x00000000, 0x000001ff },
10092                 { BUFMGR_MB_HIGH_WATER, 0x0000,
10093                         0x00000000, 0x000001ff },
10094                 { BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
10095                         0xffffffff, 0x00000000 },
10096                 { BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
10097                         0xffffffff, 0x00000000 },
10098
10099                 /* Mailbox Registers */
10100                 { GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
10101                         0x00000000, 0x000001ff },
10102                 { GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
10103                         0x00000000, 0x000001ff },
10104                 { GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
10105                         0x00000000, 0x000007ff },
10106                 { GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
10107                         0x00000000, 0x000001ff },
10108
10109                 { 0xffff, 0x0000, 0x00000000, 0x00000000 },
10110         };
10111
10112         is_5705 = is_5750 = 0;
10113         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
10114                 is_5705 = 1;
10115                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
10116                         is_5750 = 1;
10117         }
10118
10119         for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
10120                 if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
10121                         continue;
10122
10123                 if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
10124                         continue;
10125
10126                 if ((tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
10127                     (reg_tbl[i].flags & TG3_FL_NOT_5788))
10128                         continue;
10129
10130                 if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
10131                         continue;
10132
10133                 offset = (u32) reg_tbl[i].offset;
10134                 read_mask = reg_tbl[i].read_mask;
10135                 write_mask = reg_tbl[i].write_mask;
10136
10137                 /* Save the original register content */
10138                 save_val = tr32(offset);
10139
10140                 /* Determine the read-only value. */
10141                 read_val = save_val & read_mask;
10142
10143                 /* Write zero to the register, then make sure the read-only bits
10144                  * are not changed and the read/write bits are all zeros.
10145                  */
10146                 tw32(offset, 0);
10147
10148                 val = tr32(offset);
10149
10150                 /* Test the read-only and read/write bits. */
10151                 if (((val & read_mask) != read_val) || (val & write_mask))
10152                         goto out;
10153
10154                 /* Write ones to all the bits defined by RdMask and WrMask, then
10155                  * make sure the read-only bits are not changed and the
10156                  * read/write bits are all ones.
10157                  */
10158                 tw32(offset, read_mask | write_mask);
10159
10160                 val = tr32(offset);
10161
10162                 /* Test the read-only bits. */
10163                 if ((val & read_mask) != read_val)
10164                         goto out;
10165
10166                 /* Test the read/write bits. */
10167                 if ((val & write_mask) != write_mask)
10168                         goto out;
10169
10170                 tw32(offset, save_val);
10171         }
10172
10173         return 0;
10174
10175 out:
10176         if (netif_msg_hw(tp))
10177                 printk(KERN_ERR PFX "Register test failed at offset %x\n",
10178                        offset);
10179         tw32(offset, save_val);
10180         return -EIO;
10181 }
10182
10183 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
10184 {
10185         static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
10186         int i;
10187         u32 j;
10188
10189         for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
10190                 for (j = 0; j < len; j += 4) {
10191                         u32 val;
10192
10193                         tg3_write_mem(tp, offset + j, test_pattern[i]);
10194                         tg3_read_mem(tp, offset + j, &val);
10195                         if (val != test_pattern[i])
10196                                 return -EIO;
10197                 }
10198         }
10199         return 0;
10200 }
10201
10202 static int tg3_test_memory(struct tg3 *tp)
10203 {
10204         static struct mem_entry {
10205                 u32 offset;
10206                 u32 len;
10207         } mem_tbl_570x[] = {
10208                 { 0x00000000, 0x00b50},
10209                 { 0x00002000, 0x1c000},
10210                 { 0xffffffff, 0x00000}
10211         }, mem_tbl_5705[] = {
10212                 { 0x00000100, 0x0000c},
10213                 { 0x00000200, 0x00008},
10214                 { 0x00004000, 0x00800},
10215                 { 0x00006000, 0x01000},
10216                 { 0x00008000, 0x02000},
10217                 { 0x00010000, 0x0e000},
10218                 { 0xffffffff, 0x00000}
10219         }, mem_tbl_5755[] = {
10220                 { 0x00000200, 0x00008},
10221                 { 0x00004000, 0x00800},
10222                 { 0x00006000, 0x00800},
10223                 { 0x00008000, 0x02000},
10224                 { 0x00010000, 0x0c000},
10225                 { 0xffffffff, 0x00000}
10226         }, mem_tbl_5906[] = {
10227                 { 0x00000200, 0x00008},
10228                 { 0x00004000, 0x00400},
10229                 { 0x00006000, 0x00400},
10230                 { 0x00008000, 0x01000},
10231                 { 0x00010000, 0x01000},
10232                 { 0xffffffff, 0x00000}
10233         };
10234         struct mem_entry *mem_tbl;
10235         int err = 0;
10236         int i;
10237
10238         if (tp->tg3_flags3 & TG3_FLG3_5755_PLUS)
10239                 mem_tbl = mem_tbl_5755;
10240         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
10241                 mem_tbl = mem_tbl_5906;
10242         else if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS)
10243                 mem_tbl = mem_tbl_5705;
10244         else
10245                 mem_tbl = mem_tbl_570x;
10246
10247         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
10248                 if ((err = tg3_do_mem_test(tp, mem_tbl[i].offset,
10249                     mem_tbl[i].len)) != 0)
10250                         break;
10251         }
10252
10253         return err;
10254 }
10255
10256 #define TG3_MAC_LOOPBACK        0
10257 #define TG3_PHY_LOOPBACK        1
10258
10259 static int tg3_run_loopback(struct tg3 *tp, int loopback_mode)
10260 {
10261         u32 mac_mode, rx_start_idx, rx_idx, tx_idx, opaque_key;
10262         u32 desc_idx, coal_now;
10263         struct sk_buff *skb, *rx_skb;
10264         u8 *tx_data;
10265         dma_addr_t map;
10266         int num_pkts, tx_len, rx_len, i, err;
10267         struct tg3_rx_buffer_desc *desc;
10268         struct tg3_napi *tnapi, *rnapi;
10269         struct tg3_rx_prodring_set *tpr = &tp->prodring[0];
10270
10271         if (tp->irq_cnt > 1) {
10272                 tnapi = &tp->napi[1];
10273                 rnapi = &tp->napi[1];
10274         } else {
10275                 tnapi = &tp->napi[0];
10276                 rnapi = &tp->napi[0];
10277         }
10278         coal_now = tnapi->coal_now | rnapi->coal_now;
10279
10280         if (loopback_mode == TG3_MAC_LOOPBACK) {
10281                 /* HW errata - mac loopback fails in some cases on 5780.
10282                  * Normal traffic and PHY loopback are not affected by
10283                  * errata.
10284                  */
10285                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780)
10286                         return 0;
10287
10288                 mac_mode = (tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK) |
10289                            MAC_MODE_PORT_INT_LPBACK;
10290                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
10291                         mac_mode |= MAC_MODE_LINK_POLARITY;
10292                 if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
10293                         mac_mode |= MAC_MODE_PORT_MODE_MII;
10294                 else
10295                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
10296                 tw32(MAC_MODE, mac_mode);
10297         } else if (loopback_mode == TG3_PHY_LOOPBACK) {
10298                 u32 val;
10299
10300                 if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
10301                         tg3_phy_fet_toggle_apd(tp, false);
10302                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED100;
10303                 } else
10304                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED1000;
10305
10306                 tg3_phy_toggle_automdix(tp, 0);
10307
10308                 tg3_writephy(tp, MII_BMCR, val);
10309                 udelay(40);
10310
10311                 mac_mode = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
10312                 if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) {
10313                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
10314                                 tg3_writephy(tp, MII_TG3_FET_PTEST, 0x1800);
10315                         mac_mode |= MAC_MODE_PORT_MODE_MII;
10316                 } else
10317                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
10318
10319                 /* reset to prevent losing 1st rx packet intermittently */
10320                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES) {
10321                         tw32_f(MAC_RX_MODE, RX_MODE_RESET);
10322                         udelay(10);
10323                         tw32_f(MAC_RX_MODE, tp->rx_mode);
10324                 }
10325                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
10326                         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)
10327                                 mac_mode &= ~MAC_MODE_LINK_POLARITY;
10328                         else if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411)
10329                                 mac_mode |= MAC_MODE_LINK_POLARITY;
10330                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
10331                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
10332                 }
10333                 tw32(MAC_MODE, mac_mode);
10334         }
10335         else
10336                 return -EINVAL;
10337
10338         err = -EIO;
10339
10340         tx_len = 1514;
10341         skb = netdev_alloc_skb(tp->dev, tx_len);
10342         if (!skb)
10343                 return -ENOMEM;
10344
10345         tx_data = skb_put(skb, tx_len);
10346         memcpy(tx_data, tp->dev->dev_addr, 6);
10347         memset(tx_data + 6, 0x0, 8);
10348
10349         tw32(MAC_RX_MTU_SIZE, tx_len + 4);
10350
10351         for (i = 14; i < tx_len; i++)
10352                 tx_data[i] = (u8) (i & 0xff);
10353
10354         map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
10355
10356         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
10357                rnapi->coal_now);
10358
10359         udelay(10);
10360
10361         rx_start_idx = rnapi->hw_status->idx[0].rx_producer;
10362
10363         num_pkts = 0;
10364
10365         tg3_set_txd(tnapi, tnapi->tx_prod, map, tx_len, 0, 1);
10366
10367         tnapi->tx_prod++;
10368         num_pkts++;
10369
10370         tw32_tx_mbox(tnapi->prodmbox, tnapi->tx_prod);
10371         tr32_mailbox(tnapi->prodmbox);
10372
10373         udelay(10);
10374
10375         /* 250 usec to allow enough time on some 10/100 Mbps devices.  */
10376         for (i = 0; i < 25; i++) {
10377                 tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
10378                        coal_now);
10379
10380                 udelay(10);
10381
10382                 tx_idx = tnapi->hw_status->idx[0].tx_consumer;
10383                 rx_idx = rnapi->hw_status->idx[0].rx_producer;
10384                 if ((tx_idx == tnapi->tx_prod) &&
10385                     (rx_idx == (rx_start_idx + num_pkts)))
10386                         break;
10387         }
10388
10389         pci_unmap_single(tp->pdev, map, tx_len, PCI_DMA_TODEVICE);
10390         dev_kfree_skb(skb);
10391
10392         if (tx_idx != tnapi->tx_prod)
10393                 goto out;
10394
10395         if (rx_idx != rx_start_idx + num_pkts)
10396                 goto out;
10397
10398         desc = &rnapi->rx_rcb[rx_start_idx];
10399         desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
10400         opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
10401         if (opaque_key != RXD_OPAQUE_RING_STD)
10402                 goto out;
10403
10404         if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
10405             (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
10406                 goto out;
10407
10408         rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4;
10409         if (rx_len != tx_len)
10410                 goto out;
10411
10412         rx_skb = tpr->rx_std_buffers[desc_idx].skb;
10413
10414         map = pci_unmap_addr(&tpr->rx_std_buffers[desc_idx], mapping);
10415         pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len, PCI_DMA_FROMDEVICE);
10416
10417         for (i = 14; i < tx_len; i++) {
10418                 if (*(rx_skb->data + i) != (u8) (i & 0xff))
10419                         goto out;
10420         }
10421         err = 0;
10422
10423         /* tg3_free_rings will unmap and free the rx_skb */
10424 out:
10425         return err;
10426 }
10427
10428 #define TG3_MAC_LOOPBACK_FAILED         1
10429 #define TG3_PHY_LOOPBACK_FAILED         2
10430 #define TG3_LOOPBACK_FAILED             (TG3_MAC_LOOPBACK_FAILED |      \
10431                                          TG3_PHY_LOOPBACK_FAILED)
10432
10433 static int tg3_test_loopback(struct tg3 *tp)
10434 {
10435         int err = 0;
10436         u32 cpmuctrl = 0;
10437
10438         if (!netif_running(tp->dev))
10439                 return TG3_LOOPBACK_FAILED;
10440
10441         err = tg3_reset_hw(tp, 1);
10442         if (err)
10443                 return TG3_LOOPBACK_FAILED;
10444
10445         /* Turn off gphy autopowerdown. */
10446         if (tp->tg3_flags3 & TG3_FLG3_PHY_ENABLE_APD)
10447                 tg3_phy_toggle_apd(tp, false);
10448
10449         if (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) {
10450                 int i;
10451                 u32 status;
10452
10453                 tw32(TG3_CPMU_MUTEX_REQ, CPMU_MUTEX_REQ_DRIVER);
10454
10455                 /* Wait for up to 40 microseconds to acquire lock. */
10456                 for (i = 0; i < 4; i++) {
10457                         status = tr32(TG3_CPMU_MUTEX_GNT);
10458                         if (status == CPMU_MUTEX_GNT_DRIVER)
10459                                 break;
10460                         udelay(10);
10461                 }
10462
10463                 if (status != CPMU_MUTEX_GNT_DRIVER)
10464                         return TG3_LOOPBACK_FAILED;
10465
10466                 /* Turn off link-based power management. */
10467                 cpmuctrl = tr32(TG3_CPMU_CTRL);
10468                 tw32(TG3_CPMU_CTRL,
10469                      cpmuctrl & ~(CPMU_CTRL_LINK_SPEED_MODE |
10470                                   CPMU_CTRL_LINK_AWARE_MODE));
10471         }
10472
10473         if (tg3_run_loopback(tp, TG3_MAC_LOOPBACK))
10474                 err |= TG3_MAC_LOOPBACK_FAILED;
10475
10476         if (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT) {
10477                 tw32(TG3_CPMU_CTRL, cpmuctrl);
10478
10479                 /* Release the mutex */
10480                 tw32(TG3_CPMU_MUTEX_GNT, CPMU_MUTEX_GNT_DRIVER);
10481         }
10482
10483         if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) &&
10484             !(tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB)) {
10485                 if (tg3_run_loopback(tp, TG3_PHY_LOOPBACK))
10486                         err |= TG3_PHY_LOOPBACK_FAILED;
10487         }
10488
10489         /* Re-enable gphy autopowerdown. */
10490         if (tp->tg3_flags3 & TG3_FLG3_PHY_ENABLE_APD)
10491                 tg3_phy_toggle_apd(tp, true);
10492
10493         return err;
10494 }
10495
10496 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
10497                           u64 *data)
10498 {
10499         struct tg3 *tp = netdev_priv(dev);
10500
10501         if (tp->link_config.phy_is_low_power)
10502                 tg3_set_power_state(tp, PCI_D0);
10503
10504         memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
10505
10506         if (tg3_test_nvram(tp) != 0) {
10507                 etest->flags |= ETH_TEST_FL_FAILED;
10508                 data[0] = 1;
10509         }
10510         if (tg3_test_link(tp) != 0) {
10511                 etest->flags |= ETH_TEST_FL_FAILED;
10512                 data[1] = 1;
10513         }
10514         if (etest->flags & ETH_TEST_FL_OFFLINE) {
10515                 int err, err2 = 0, irq_sync = 0;
10516
10517                 if (netif_running(dev)) {
10518                         tg3_phy_stop(tp);
10519                         tg3_netif_stop(tp);
10520                         irq_sync = 1;
10521                 }
10522
10523                 tg3_full_lock(tp, irq_sync);
10524
10525                 tg3_halt(tp, RESET_KIND_SUSPEND, 1);
10526                 err = tg3_nvram_lock(tp);
10527                 tg3_halt_cpu(tp, RX_CPU_BASE);
10528                 if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
10529                         tg3_halt_cpu(tp, TX_CPU_BASE);
10530                 if (!err)
10531                         tg3_nvram_unlock(tp);
10532
10533                 if (tp->tg3_flags2 & TG3_FLG2_MII_SERDES)
10534                         tg3_phy_reset(tp);
10535
10536                 if (tg3_test_registers(tp) != 0) {
10537                         etest->flags |= ETH_TEST_FL_FAILED;
10538                         data[2] = 1;
10539                 }
10540                 if (tg3_test_memory(tp) != 0) {
10541                         etest->flags |= ETH_TEST_FL_FAILED;
10542                         data[3] = 1;
10543                 }
10544                 if ((data[4] = tg3_test_loopback(tp)) != 0)
10545                         etest->flags |= ETH_TEST_FL_FAILED;
10546
10547                 tg3_full_unlock(tp);
10548
10549                 if (tg3_test_interrupt(tp) != 0) {
10550                         etest->flags |= ETH_TEST_FL_FAILED;
10551                         data[5] = 1;
10552                 }
10553
10554                 tg3_full_lock(tp, 0);
10555
10556                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
10557                 if (netif_running(dev)) {
10558                         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
10559                         err2 = tg3_restart_hw(tp, 1);
10560                         if (!err2)
10561                                 tg3_netif_start(tp);
10562                 }
10563
10564                 tg3_full_unlock(tp);
10565
10566                 if (irq_sync && !err2)
10567                         tg3_phy_start(tp);
10568         }
10569         if (tp->link_config.phy_is_low_power)
10570                 tg3_set_power_state(tp, PCI_D3hot);
10571
10572 }
10573
10574 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10575 {
10576         struct mii_ioctl_data *data = if_mii(ifr);
10577         struct tg3 *tp = netdev_priv(dev);
10578         int err;
10579
10580         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
10581                 if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED))
10582                         return -EAGAIN;
10583                 return phy_mii_ioctl(tp->mdio_bus->phy_map[PHY_ADDR], data, cmd);
10584         }
10585
10586         switch(cmd) {
10587         case SIOCGMIIPHY:
10588                 data->phy_id = tp->phy_addr;
10589
10590                 /* fallthru */
10591         case SIOCGMIIREG: {
10592                 u32 mii_regval;
10593
10594                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
10595                         break;                  /* We have no PHY */
10596
10597                 if (tp->link_config.phy_is_low_power)
10598                         return -EAGAIN;
10599
10600                 spin_lock_bh(&tp->lock);
10601                 err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval);
10602                 spin_unlock_bh(&tp->lock);
10603
10604                 data->val_out = mii_regval;
10605
10606                 return err;
10607         }
10608
10609         case SIOCSMIIREG:
10610                 if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
10611                         break;                  /* We have no PHY */
10612
10613                 if (tp->link_config.phy_is_low_power)
10614                         return -EAGAIN;
10615
10616                 spin_lock_bh(&tp->lock);
10617                 err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in);
10618                 spin_unlock_bh(&tp->lock);
10619
10620                 return err;
10621
10622         default:
10623                 /* do nothing */
10624                 break;
10625         }
10626         return -EOPNOTSUPP;
10627 }
10628
10629 #if TG3_VLAN_TAG_USED
10630 static void tg3_vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
10631 {
10632         struct tg3 *tp = netdev_priv(dev);
10633
10634         if (!netif_running(dev)) {
10635                 tp->vlgrp = grp;
10636                 return;
10637         }
10638
10639         tg3_netif_stop(tp);
10640
10641         tg3_full_lock(tp, 0);
10642
10643         tp->vlgrp = grp;
10644
10645         /* Update RX_MODE_KEEP_VLAN_TAG bit in RX_MODE register. */
10646         __tg3_set_rx_mode(dev);
10647
10648         tg3_netif_start(tp);
10649
10650         tg3_full_unlock(tp);
10651 }
10652 #endif
10653
10654 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
10655 {
10656         struct tg3 *tp = netdev_priv(dev);
10657
10658         memcpy(ec, &tp->coal, sizeof(*ec));
10659         return 0;
10660 }
10661
10662 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
10663 {
10664         struct tg3 *tp = netdev_priv(dev);
10665         u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
10666         u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
10667
10668         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS)) {
10669                 max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
10670                 max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
10671                 max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
10672                 min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
10673         }
10674
10675         if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
10676             (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
10677             (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
10678             (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
10679             (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
10680             (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
10681             (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
10682             (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
10683             (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
10684             (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
10685                 return -EINVAL;
10686
10687         /* No rx interrupts will be generated if both are zero */
10688         if ((ec->rx_coalesce_usecs == 0) &&
10689             (ec->rx_max_coalesced_frames == 0))
10690                 return -EINVAL;
10691
10692         /* No tx interrupts will be generated if both are zero */
10693         if ((ec->tx_coalesce_usecs == 0) &&
10694             (ec->tx_max_coalesced_frames == 0))
10695                 return -EINVAL;
10696
10697         /* Only copy relevant parameters, ignore all others. */
10698         tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
10699         tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
10700         tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
10701         tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
10702         tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
10703         tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
10704         tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
10705         tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
10706         tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
10707
10708         if (netif_running(dev)) {
10709                 tg3_full_lock(tp, 0);
10710                 __tg3_set_coalesce(tp, &tp->coal);
10711                 tg3_full_unlock(tp);
10712         }
10713         return 0;
10714 }
10715
10716 static const struct ethtool_ops tg3_ethtool_ops = {
10717         .get_settings           = tg3_get_settings,
10718         .set_settings           = tg3_set_settings,
10719         .get_drvinfo            = tg3_get_drvinfo,
10720         .get_regs_len           = tg3_get_regs_len,
10721         .get_regs               = tg3_get_regs,
10722         .get_wol                = tg3_get_wol,
10723         .set_wol                = tg3_set_wol,
10724         .get_msglevel           = tg3_get_msglevel,
10725         .set_msglevel           = tg3_set_msglevel,
10726         .nway_reset             = tg3_nway_reset,
10727         .get_link               = ethtool_op_get_link,
10728         .get_eeprom_len         = tg3_get_eeprom_len,
10729         .get_eeprom             = tg3_get_eeprom,
10730         .set_eeprom             = tg3_set_eeprom,
10731         .get_ringparam          = tg3_get_ringparam,
10732         .set_ringparam          = tg3_set_ringparam,
10733         .get_pauseparam         = tg3_get_pauseparam,
10734         .set_pauseparam         = tg3_set_pauseparam,
10735         .get_rx_csum            = tg3_get_rx_csum,
10736         .set_rx_csum            = tg3_set_rx_csum,
10737         .set_tx_csum            = tg3_set_tx_csum,
10738         .set_sg                 = ethtool_op_set_sg,
10739         .set_tso                = tg3_set_tso,
10740         .self_test              = tg3_self_test,
10741         .get_strings            = tg3_get_strings,
10742         .phys_id                = tg3_phys_id,
10743         .get_ethtool_stats      = tg3_get_ethtool_stats,
10744         .get_coalesce           = tg3_get_coalesce,
10745         .set_coalesce           = tg3_set_coalesce,
10746         .get_sset_count         = tg3_get_sset_count,
10747 };
10748
10749 static void __devinit tg3_get_eeprom_size(struct tg3 *tp)
10750 {
10751         u32 cursize, val, magic;
10752
10753         tp->nvram_size = EEPROM_CHIP_SIZE;
10754
10755         if (tg3_nvram_read(tp, 0, &magic) != 0)
10756                 return;
10757
10758         if ((magic != TG3_EEPROM_MAGIC) &&
10759             ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
10760             ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
10761                 return;
10762
10763         /*
10764          * Size the chip by reading offsets at increasing powers of two.
10765          * When we encounter our validation signature, we know the addressing
10766          * has wrapped around, and thus have our chip size.
10767          */
10768         cursize = 0x10;
10769
10770         while (cursize < tp->nvram_size) {
10771                 if (tg3_nvram_read(tp, cursize, &val) != 0)
10772                         return;
10773
10774                 if (val == magic)
10775                         break;
10776
10777                 cursize <<= 1;
10778         }
10779
10780         tp->nvram_size = cursize;
10781 }
10782
10783 static void __devinit tg3_get_nvram_size(struct tg3 *tp)
10784 {
10785         u32 val;
10786
10787         if ((tp->tg3_flags3 & TG3_FLG3_NO_NVRAM) ||
10788             tg3_nvram_read(tp, 0, &val) != 0)
10789                 return;
10790
10791         /* Selfboot format */
10792         if (val != TG3_EEPROM_MAGIC) {
10793                 tg3_get_eeprom_size(tp);
10794                 return;
10795         }
10796
10797         if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
10798                 if (val != 0) {
10799                         /* This is confusing.  We want to operate on the
10800                          * 16-bit value at offset 0xf2.  The tg3_nvram_read()
10801                          * call will read from NVRAM and byteswap the data
10802                          * according to the byteswapping settings for all
10803                          * other register accesses.  This ensures the data we
10804                          * want will always reside in the lower 16-bits.
10805                          * However, the data in NVRAM is in LE format, which
10806                          * means the data from the NVRAM read will always be
10807                          * opposite the endianness of the CPU.  The 16-bit
10808                          * byteswap then brings the data to CPU endianness.
10809                          */
10810                         tp->nvram_size = swab16((u16)(val & 0x0000ffff)) * 1024;
10811                         return;
10812                 }
10813         }
10814         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
10815 }
10816
10817 static void __devinit tg3_get_nvram_info(struct tg3 *tp)
10818 {
10819         u32 nvcfg1;
10820
10821         nvcfg1 = tr32(NVRAM_CFG1);
10822         if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
10823                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
10824         } else {
10825                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
10826                 tw32(NVRAM_CFG1, nvcfg1);
10827         }
10828
10829         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) ||
10830             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
10831                 switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
10832                 case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
10833                         tp->nvram_jedecnum = JEDEC_ATMEL;
10834                         tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
10835                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10836                         break;
10837                 case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
10838                         tp->nvram_jedecnum = JEDEC_ATMEL;
10839                         tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
10840                         break;
10841                 case FLASH_VENDOR_ATMEL_EEPROM:
10842                         tp->nvram_jedecnum = JEDEC_ATMEL;
10843                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
10844                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10845                         break;
10846                 case FLASH_VENDOR_ST:
10847                         tp->nvram_jedecnum = JEDEC_ST;
10848                         tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
10849                         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10850                         break;
10851                 case FLASH_VENDOR_SAIFUN:
10852                         tp->nvram_jedecnum = JEDEC_SAIFUN;
10853                         tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
10854                         break;
10855                 case FLASH_VENDOR_SST_SMALL:
10856                 case FLASH_VENDOR_SST_LARGE:
10857                         tp->nvram_jedecnum = JEDEC_SST;
10858                         tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
10859                         break;
10860                 }
10861         } else {
10862                 tp->nvram_jedecnum = JEDEC_ATMEL;
10863                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
10864                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10865         }
10866 }
10867
10868 static void __devinit tg3_nvram_get_pagesize(struct tg3 *tp, u32 nvmcfg1)
10869 {
10870         switch (nvmcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
10871         case FLASH_5752PAGE_SIZE_256:
10872                 tp->nvram_pagesize = 256;
10873                 break;
10874         case FLASH_5752PAGE_SIZE_512:
10875                 tp->nvram_pagesize = 512;
10876                 break;
10877         case FLASH_5752PAGE_SIZE_1K:
10878                 tp->nvram_pagesize = 1024;
10879                 break;
10880         case FLASH_5752PAGE_SIZE_2K:
10881                 tp->nvram_pagesize = 2048;
10882                 break;
10883         case FLASH_5752PAGE_SIZE_4K:
10884                 tp->nvram_pagesize = 4096;
10885                 break;
10886         case FLASH_5752PAGE_SIZE_264:
10887                 tp->nvram_pagesize = 264;
10888                 break;
10889         case FLASH_5752PAGE_SIZE_528:
10890                 tp->nvram_pagesize = 528;
10891                 break;
10892         }
10893 }
10894
10895 static void __devinit tg3_get_5752_nvram_info(struct tg3 *tp)
10896 {
10897         u32 nvcfg1;
10898
10899         nvcfg1 = tr32(NVRAM_CFG1);
10900
10901         /* NVRAM protection for TPM */
10902         if (nvcfg1 & (1 << 27))
10903                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
10904
10905         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
10906         case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
10907         case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
10908                 tp->nvram_jedecnum = JEDEC_ATMEL;
10909                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10910                 break;
10911         case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
10912                 tp->nvram_jedecnum = JEDEC_ATMEL;
10913                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10914                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
10915                 break;
10916         case FLASH_5752VENDOR_ST_M45PE10:
10917         case FLASH_5752VENDOR_ST_M45PE20:
10918         case FLASH_5752VENDOR_ST_M45PE40:
10919                 tp->nvram_jedecnum = JEDEC_ST;
10920                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10921                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
10922                 break;
10923         }
10924
10925         if (tp->tg3_flags2 & TG3_FLG2_FLASH) {
10926                 tg3_nvram_get_pagesize(tp, nvcfg1);
10927         } else {
10928                 /* For eeprom, set pagesize to maximum eeprom size */
10929                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
10930
10931                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
10932                 tw32(NVRAM_CFG1, nvcfg1);
10933         }
10934 }
10935
10936 static void __devinit tg3_get_5755_nvram_info(struct tg3 *tp)
10937 {
10938         u32 nvcfg1, protect = 0;
10939
10940         nvcfg1 = tr32(NVRAM_CFG1);
10941
10942         /* NVRAM protection for TPM */
10943         if (nvcfg1 & (1 << 27)) {
10944                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
10945                 protect = 1;
10946         }
10947
10948         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
10949         switch (nvcfg1) {
10950         case FLASH_5755VENDOR_ATMEL_FLASH_1:
10951         case FLASH_5755VENDOR_ATMEL_FLASH_2:
10952         case FLASH_5755VENDOR_ATMEL_FLASH_3:
10953         case FLASH_5755VENDOR_ATMEL_FLASH_5:
10954                 tp->nvram_jedecnum = JEDEC_ATMEL;
10955                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10956                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
10957                 tp->nvram_pagesize = 264;
10958                 if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
10959                     nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
10960                         tp->nvram_size = (protect ? 0x3e200 :
10961                                           TG3_NVRAM_SIZE_512KB);
10962                 else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
10963                         tp->nvram_size = (protect ? 0x1f200 :
10964                                           TG3_NVRAM_SIZE_256KB);
10965                 else
10966                         tp->nvram_size = (protect ? 0x1f200 :
10967                                           TG3_NVRAM_SIZE_128KB);
10968                 break;
10969         case FLASH_5752VENDOR_ST_M45PE10:
10970         case FLASH_5752VENDOR_ST_M45PE20:
10971         case FLASH_5752VENDOR_ST_M45PE40:
10972                 tp->nvram_jedecnum = JEDEC_ST;
10973                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
10974                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
10975                 tp->nvram_pagesize = 256;
10976                 if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
10977                         tp->nvram_size = (protect ?
10978                                           TG3_NVRAM_SIZE_64KB :
10979                                           TG3_NVRAM_SIZE_128KB);
10980                 else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
10981                         tp->nvram_size = (protect ?
10982                                           TG3_NVRAM_SIZE_64KB :
10983                                           TG3_NVRAM_SIZE_256KB);
10984                 else
10985                         tp->nvram_size = (protect ?
10986                                           TG3_NVRAM_SIZE_128KB :
10987                                           TG3_NVRAM_SIZE_512KB);
10988                 break;
10989         }
10990 }
10991
10992 static void __devinit tg3_get_5787_nvram_info(struct tg3 *tp)
10993 {
10994         u32 nvcfg1;
10995
10996         nvcfg1 = tr32(NVRAM_CFG1);
10997
10998         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
10999         case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
11000         case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
11001         case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
11002         case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
11003                 tp->nvram_jedecnum = JEDEC_ATMEL;
11004                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11005                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11006
11007                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11008                 tw32(NVRAM_CFG1, nvcfg1);
11009                 break;
11010         case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
11011         case FLASH_5755VENDOR_ATMEL_FLASH_1:
11012         case FLASH_5755VENDOR_ATMEL_FLASH_2:
11013         case FLASH_5755VENDOR_ATMEL_FLASH_3:
11014                 tp->nvram_jedecnum = JEDEC_ATMEL;
11015                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11016                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11017                 tp->nvram_pagesize = 264;
11018                 break;
11019         case FLASH_5752VENDOR_ST_M45PE10:
11020         case FLASH_5752VENDOR_ST_M45PE20:
11021         case FLASH_5752VENDOR_ST_M45PE40:
11022                 tp->nvram_jedecnum = JEDEC_ST;
11023                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11024                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11025                 tp->nvram_pagesize = 256;
11026                 break;
11027         }
11028 }
11029
11030 static void __devinit tg3_get_5761_nvram_info(struct tg3 *tp)
11031 {
11032         u32 nvcfg1, protect = 0;
11033
11034         nvcfg1 = tr32(NVRAM_CFG1);
11035
11036         /* NVRAM protection for TPM */
11037         if (nvcfg1 & (1 << 27)) {
11038                 tp->tg3_flags2 |= TG3_FLG2_PROTECTED_NVRAM;
11039                 protect = 1;
11040         }
11041
11042         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
11043         switch (nvcfg1) {
11044         case FLASH_5761VENDOR_ATMEL_ADB021D:
11045         case FLASH_5761VENDOR_ATMEL_ADB041D:
11046         case FLASH_5761VENDOR_ATMEL_ADB081D:
11047         case FLASH_5761VENDOR_ATMEL_ADB161D:
11048         case FLASH_5761VENDOR_ATMEL_MDB021D:
11049         case FLASH_5761VENDOR_ATMEL_MDB041D:
11050         case FLASH_5761VENDOR_ATMEL_MDB081D:
11051         case FLASH_5761VENDOR_ATMEL_MDB161D:
11052                 tp->nvram_jedecnum = JEDEC_ATMEL;
11053                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11054                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11055                 tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM_ADDR_TRANS;
11056                 tp->nvram_pagesize = 256;
11057                 break;
11058         case FLASH_5761VENDOR_ST_A_M45PE20:
11059         case FLASH_5761VENDOR_ST_A_M45PE40:
11060         case FLASH_5761VENDOR_ST_A_M45PE80:
11061         case FLASH_5761VENDOR_ST_A_M45PE16:
11062         case FLASH_5761VENDOR_ST_M_M45PE20:
11063         case FLASH_5761VENDOR_ST_M_M45PE40:
11064         case FLASH_5761VENDOR_ST_M_M45PE80:
11065         case FLASH_5761VENDOR_ST_M_M45PE16:
11066                 tp->nvram_jedecnum = JEDEC_ST;
11067                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11068                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11069                 tp->nvram_pagesize = 256;
11070                 break;
11071         }
11072
11073         if (protect) {
11074                 tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
11075         } else {
11076                 switch (nvcfg1) {
11077                 case FLASH_5761VENDOR_ATMEL_ADB161D:
11078                 case FLASH_5761VENDOR_ATMEL_MDB161D:
11079                 case FLASH_5761VENDOR_ST_A_M45PE16:
11080                 case FLASH_5761VENDOR_ST_M_M45PE16:
11081                         tp->nvram_size = TG3_NVRAM_SIZE_2MB;
11082                         break;
11083                 case FLASH_5761VENDOR_ATMEL_ADB081D:
11084                 case FLASH_5761VENDOR_ATMEL_MDB081D:
11085                 case FLASH_5761VENDOR_ST_A_M45PE80:
11086                 case FLASH_5761VENDOR_ST_M_M45PE80:
11087                         tp->nvram_size = TG3_NVRAM_SIZE_1MB;
11088                         break;
11089                 case FLASH_5761VENDOR_ATMEL_ADB041D:
11090                 case FLASH_5761VENDOR_ATMEL_MDB041D:
11091                 case FLASH_5761VENDOR_ST_A_M45PE40:
11092                 case FLASH_5761VENDOR_ST_M_M45PE40:
11093                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
11094                         break;
11095                 case FLASH_5761VENDOR_ATMEL_ADB021D:
11096                 case FLASH_5761VENDOR_ATMEL_MDB021D:
11097                 case FLASH_5761VENDOR_ST_A_M45PE20:
11098                 case FLASH_5761VENDOR_ST_M_M45PE20:
11099                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11100                         break;
11101                 }
11102         }
11103 }
11104
11105 static void __devinit tg3_get_5906_nvram_info(struct tg3 *tp)
11106 {
11107         tp->nvram_jedecnum = JEDEC_ATMEL;
11108         tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11109         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11110 }
11111
11112 static void __devinit tg3_get_57780_nvram_info(struct tg3 *tp)
11113 {
11114         u32 nvcfg1;
11115
11116         nvcfg1 = tr32(NVRAM_CFG1);
11117
11118         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11119         case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
11120         case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
11121                 tp->nvram_jedecnum = JEDEC_ATMEL;
11122                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11123                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11124
11125                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11126                 tw32(NVRAM_CFG1, nvcfg1);
11127                 return;
11128         case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
11129         case FLASH_57780VENDOR_ATMEL_AT45DB011D:
11130         case FLASH_57780VENDOR_ATMEL_AT45DB011B:
11131         case FLASH_57780VENDOR_ATMEL_AT45DB021D:
11132         case FLASH_57780VENDOR_ATMEL_AT45DB021B:
11133         case FLASH_57780VENDOR_ATMEL_AT45DB041D:
11134         case FLASH_57780VENDOR_ATMEL_AT45DB041B:
11135                 tp->nvram_jedecnum = JEDEC_ATMEL;
11136                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11137                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11138
11139                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11140                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
11141                 case FLASH_57780VENDOR_ATMEL_AT45DB011D:
11142                 case FLASH_57780VENDOR_ATMEL_AT45DB011B:
11143                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
11144                         break;
11145                 case FLASH_57780VENDOR_ATMEL_AT45DB021D:
11146                 case FLASH_57780VENDOR_ATMEL_AT45DB021B:
11147                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11148                         break;
11149                 case FLASH_57780VENDOR_ATMEL_AT45DB041D:
11150                 case FLASH_57780VENDOR_ATMEL_AT45DB041B:
11151                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
11152                         break;
11153                 }
11154                 break;
11155         case FLASH_5752VENDOR_ST_M45PE10:
11156         case FLASH_5752VENDOR_ST_M45PE20:
11157         case FLASH_5752VENDOR_ST_M45PE40:
11158                 tp->nvram_jedecnum = JEDEC_ST;
11159                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11160                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11161
11162                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11163                 case FLASH_5752VENDOR_ST_M45PE10:
11164                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
11165                         break;
11166                 case FLASH_5752VENDOR_ST_M45PE20:
11167                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11168                         break;
11169                 case FLASH_5752VENDOR_ST_M45PE40:
11170                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
11171                         break;
11172                 }
11173                 break;
11174         default:
11175                 tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM;
11176                 return;
11177         }
11178
11179         tg3_nvram_get_pagesize(tp, nvcfg1);
11180         if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
11181                 tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM_ADDR_TRANS;
11182 }
11183
11184
11185 static void __devinit tg3_get_5717_nvram_info(struct tg3 *tp)
11186 {
11187         u32 nvcfg1;
11188
11189         nvcfg1 = tr32(NVRAM_CFG1);
11190
11191         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11192         case FLASH_5717VENDOR_ATMEL_EEPROM:
11193         case FLASH_5717VENDOR_MICRO_EEPROM:
11194                 tp->nvram_jedecnum = JEDEC_ATMEL;
11195                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11196                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11197
11198                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11199                 tw32(NVRAM_CFG1, nvcfg1);
11200                 return;
11201         case FLASH_5717VENDOR_ATMEL_MDB011D:
11202         case FLASH_5717VENDOR_ATMEL_ADB011B:
11203         case FLASH_5717VENDOR_ATMEL_ADB011D:
11204         case FLASH_5717VENDOR_ATMEL_MDB021D:
11205         case FLASH_5717VENDOR_ATMEL_ADB021B:
11206         case FLASH_5717VENDOR_ATMEL_ADB021D:
11207         case FLASH_5717VENDOR_ATMEL_45USPT:
11208                 tp->nvram_jedecnum = JEDEC_ATMEL;
11209                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11210                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11211
11212                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11213                 case FLASH_5717VENDOR_ATMEL_MDB021D:
11214                 case FLASH_5717VENDOR_ATMEL_ADB021B:
11215                 case FLASH_5717VENDOR_ATMEL_ADB021D:
11216                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11217                         break;
11218                 default:
11219                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
11220                         break;
11221                 }
11222                 break;
11223         case FLASH_5717VENDOR_ST_M_M25PE10:
11224         case FLASH_5717VENDOR_ST_A_M25PE10:
11225         case FLASH_5717VENDOR_ST_M_M45PE10:
11226         case FLASH_5717VENDOR_ST_A_M45PE10:
11227         case FLASH_5717VENDOR_ST_M_M25PE20:
11228         case FLASH_5717VENDOR_ST_A_M25PE20:
11229         case FLASH_5717VENDOR_ST_M_M45PE20:
11230         case FLASH_5717VENDOR_ST_A_M45PE20:
11231         case FLASH_5717VENDOR_ST_25USPT:
11232         case FLASH_5717VENDOR_ST_45USPT:
11233                 tp->nvram_jedecnum = JEDEC_ST;
11234                 tp->tg3_flags |= TG3_FLAG_NVRAM_BUFFERED;
11235                 tp->tg3_flags2 |= TG3_FLG2_FLASH;
11236
11237                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11238                 case FLASH_5717VENDOR_ST_M_M25PE20:
11239                 case FLASH_5717VENDOR_ST_A_M25PE20:
11240                 case FLASH_5717VENDOR_ST_M_M45PE20:
11241                 case FLASH_5717VENDOR_ST_A_M45PE20:
11242                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11243                         break;
11244                 default:
11245                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
11246                         break;
11247                 }
11248                 break;
11249         default:
11250                 tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM;
11251                 return;
11252         }
11253
11254         tg3_nvram_get_pagesize(tp, nvcfg1);
11255         if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
11256                 tp->tg3_flags3 |= TG3_FLG3_NO_NVRAM_ADDR_TRANS;
11257 }
11258
11259 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
11260 static void __devinit tg3_nvram_init(struct tg3 *tp)
11261 {
11262         tw32_f(GRC_EEPROM_ADDR,
11263              (EEPROM_ADDR_FSM_RESET |
11264               (EEPROM_DEFAULT_CLOCK_PERIOD <<
11265                EEPROM_ADDR_CLKPERD_SHIFT)));
11266
11267         msleep(1);
11268
11269         /* Enable seeprom accesses. */
11270         tw32_f(GRC_LOCAL_CTRL,
11271              tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
11272         udelay(100);
11273
11274         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
11275             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
11276                 tp->tg3_flags |= TG3_FLAG_NVRAM;
11277
11278                 if (tg3_nvram_lock(tp)) {
11279                         printk(KERN_WARNING PFX "%s: Cannot get nvarm lock, "
11280                                "tg3_nvram_init failed.\n", tp->dev->name);
11281                         return;
11282                 }
11283                 tg3_enable_nvram_access(tp);
11284
11285                 tp->nvram_size = 0;
11286
11287                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
11288                         tg3_get_5752_nvram_info(tp);
11289                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
11290                         tg3_get_5755_nvram_info(tp);
11291                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
11292                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
11293                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
11294                         tg3_get_5787_nvram_info(tp);
11295                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
11296                         tg3_get_5761_nvram_info(tp);
11297                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
11298                         tg3_get_5906_nvram_info(tp);
11299                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
11300                         tg3_get_57780_nvram_info(tp);
11301                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
11302                         tg3_get_5717_nvram_info(tp);
11303                 else
11304                         tg3_get_nvram_info(tp);
11305
11306                 if (tp->nvram_size == 0)
11307                         tg3_get_nvram_size(tp);
11308
11309                 tg3_disable_nvram_access(tp);
11310                 tg3_nvram_unlock(tp);
11311
11312         } else {
11313                 tp->tg3_flags &= ~(TG3_FLAG_NVRAM | TG3_FLAG_NVRAM_BUFFERED);
11314
11315                 tg3_get_eeprom_size(tp);
11316         }
11317 }
11318
11319 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
11320                                     u32 offset, u32 len, u8 *buf)
11321 {
11322         int i, j, rc = 0;
11323         u32 val;
11324
11325         for (i = 0; i < len; i += 4) {
11326                 u32 addr;
11327                 __be32 data;
11328
11329                 addr = offset + i;
11330
11331                 memcpy(&data, buf + i, 4);
11332
11333                 /*
11334                  * The SEEPROM interface expects the data to always be opposite
11335                  * the native endian format.  We accomplish this by reversing
11336                  * all the operations that would have been performed on the
11337                  * data from a call to tg3_nvram_read_be32().
11338                  */
11339                 tw32(GRC_EEPROM_DATA, swab32(be32_to_cpu(data)));
11340
11341                 val = tr32(GRC_EEPROM_ADDR);
11342                 tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
11343
11344                 val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
11345                         EEPROM_ADDR_READ);
11346                 tw32(GRC_EEPROM_ADDR, val |
11347                         (0 << EEPROM_ADDR_DEVID_SHIFT) |
11348                         (addr & EEPROM_ADDR_ADDR_MASK) |
11349                         EEPROM_ADDR_START |
11350                         EEPROM_ADDR_WRITE);
11351
11352                 for (j = 0; j < 1000; j++) {
11353                         val = tr32(GRC_EEPROM_ADDR);
11354
11355                         if (val & EEPROM_ADDR_COMPLETE)
11356                                 break;
11357                         msleep(1);
11358                 }
11359                 if (!(val & EEPROM_ADDR_COMPLETE)) {
11360                         rc = -EBUSY;
11361                         break;
11362                 }
11363         }
11364
11365         return rc;
11366 }
11367
11368 /* offset and length are dword aligned */
11369 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
11370                 u8 *buf)
11371 {
11372         int ret = 0;
11373         u32 pagesize = tp->nvram_pagesize;
11374         u32 pagemask = pagesize - 1;
11375         u32 nvram_cmd;
11376         u8 *tmp;
11377
11378         tmp = kmalloc(pagesize, GFP_KERNEL);
11379         if (tmp == NULL)
11380                 return -ENOMEM;
11381
11382         while (len) {
11383                 int j;
11384                 u32 phy_addr, page_off, size;
11385
11386                 phy_addr = offset & ~pagemask;
11387
11388                 for (j = 0; j < pagesize; j += 4) {
11389                         ret = tg3_nvram_read_be32(tp, phy_addr + j,
11390                                                   (__be32 *) (tmp + j));
11391                         if (ret)
11392                                 break;
11393                 }
11394                 if (ret)
11395                         break;
11396
11397                 page_off = offset & pagemask;
11398                 size = pagesize;
11399                 if (len < size)
11400                         size = len;
11401
11402                 len -= size;
11403
11404                 memcpy(tmp + page_off, buf, size);
11405
11406                 offset = offset + (pagesize - page_off);
11407
11408                 tg3_enable_nvram_access(tp);
11409
11410                 /*
11411                  * Before we can erase the flash page, we need
11412                  * to issue a special "write enable" command.
11413                  */
11414                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
11415
11416                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
11417                         break;
11418
11419                 /* Erase the target page */
11420                 tw32(NVRAM_ADDR, phy_addr);
11421
11422                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
11423                         NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
11424
11425                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
11426                         break;
11427
11428                 /* Issue another write enable to start the write. */
11429                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
11430
11431                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
11432                         break;
11433
11434                 for (j = 0; j < pagesize; j += 4) {
11435                         __be32 data;
11436
11437                         data = *((__be32 *) (tmp + j));
11438
11439                         tw32(NVRAM_WRDATA, be32_to_cpu(data));
11440
11441                         tw32(NVRAM_ADDR, phy_addr + j);
11442
11443                         nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
11444                                 NVRAM_CMD_WR;
11445
11446                         if (j == 0)
11447                                 nvram_cmd |= NVRAM_CMD_FIRST;
11448                         else if (j == (pagesize - 4))
11449                                 nvram_cmd |= NVRAM_CMD_LAST;
11450
11451                         if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
11452                                 break;
11453                 }
11454                 if (ret)
11455                         break;
11456         }
11457
11458         nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
11459         tg3_nvram_exec_cmd(tp, nvram_cmd);
11460
11461         kfree(tmp);
11462
11463         return ret;
11464 }
11465
11466 /* offset and length are dword aligned */
11467 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
11468                 u8 *buf)
11469 {
11470         int i, ret = 0;
11471
11472         for (i = 0; i < len; i += 4, offset += 4) {
11473                 u32 page_off, phy_addr, nvram_cmd;
11474                 __be32 data;
11475
11476                 memcpy(&data, buf + i, 4);
11477                 tw32(NVRAM_WRDATA, be32_to_cpu(data));
11478
11479                 page_off = offset % tp->nvram_pagesize;
11480
11481                 phy_addr = tg3_nvram_phys_addr(tp, offset);
11482
11483                 tw32(NVRAM_ADDR, phy_addr);
11484
11485                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
11486
11487                 if ((page_off == 0) || (i == 0))
11488                         nvram_cmd |= NVRAM_CMD_FIRST;
11489                 if (page_off == (tp->nvram_pagesize - 4))
11490                         nvram_cmd |= NVRAM_CMD_LAST;
11491
11492                 if (i == (len - 4))
11493                         nvram_cmd |= NVRAM_CMD_LAST;
11494
11495                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752 &&
11496                     !(tp->tg3_flags3 & TG3_FLG3_5755_PLUS) &&
11497                     (tp->nvram_jedecnum == JEDEC_ST) &&
11498                     (nvram_cmd & NVRAM_CMD_FIRST)) {
11499
11500                         if ((ret = tg3_nvram_exec_cmd(tp,
11501                                 NVRAM_CMD_WREN | NVRAM_CMD_GO |
11502                                 NVRAM_CMD_DONE)))
11503
11504                                 break;
11505                 }
11506                 if (!(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
11507                         /* We always do complete word writes to eeprom. */
11508                         nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
11509                 }
11510
11511                 if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
11512                         break;
11513         }
11514         return ret;
11515 }
11516
11517 /* offset and length are dword aligned */
11518 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
11519 {
11520         int ret;
11521
11522         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
11523                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
11524                        ~GRC_LCLCTRL_GPIO_OUTPUT1);
11525                 udelay(40);
11526         }
11527
11528         if (!(tp->tg3_flags & TG3_FLAG_NVRAM)) {
11529                 ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
11530         }
11531         else {
11532                 u32 grc_mode;
11533
11534                 ret = tg3_nvram_lock(tp);
11535                 if (ret)
11536                         return ret;
11537
11538                 tg3_enable_nvram_access(tp);
11539                 if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) &&
11540                     !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM))
11541                         tw32(NVRAM_WRITE1, 0x406);
11542
11543                 grc_mode = tr32(GRC_MODE);
11544                 tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
11545
11546                 if ((tp->tg3_flags & TG3_FLAG_NVRAM_BUFFERED) ||
11547                         !(tp->tg3_flags2 & TG3_FLG2_FLASH)) {
11548
11549                         ret = tg3_nvram_write_block_buffered(tp, offset, len,
11550                                 buf);
11551                 }
11552                 else {
11553                         ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
11554                                 buf);
11555                 }
11556
11557                 grc_mode = tr32(GRC_MODE);
11558                 tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
11559
11560                 tg3_disable_nvram_access(tp);
11561                 tg3_nvram_unlock(tp);
11562         }
11563
11564         if (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) {
11565                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
11566                 udelay(40);
11567         }
11568
11569         return ret;
11570 }
11571
11572 struct subsys_tbl_ent {
11573         u16 subsys_vendor, subsys_devid;
11574         u32 phy_id;
11575 };
11576
11577 static struct subsys_tbl_ent subsys_id_to_phy_id[] = {
11578         /* Broadcom boards. */
11579         { PCI_VENDOR_ID_BROADCOM, 0x1644, PHY_ID_BCM5401 }, /* BCM95700A6 */
11580         { PCI_VENDOR_ID_BROADCOM, 0x0001, PHY_ID_BCM5701 }, /* BCM95701A5 */
11581         { PCI_VENDOR_ID_BROADCOM, 0x0002, PHY_ID_BCM8002 }, /* BCM95700T6 */
11582         { PCI_VENDOR_ID_BROADCOM, 0x0003, 0 },              /* BCM95700A9 */
11583         { PCI_VENDOR_ID_BROADCOM, 0x0005, PHY_ID_BCM5701 }, /* BCM95701T1 */
11584         { PCI_VENDOR_ID_BROADCOM, 0x0006, PHY_ID_BCM5701 }, /* BCM95701T8 */
11585         { PCI_VENDOR_ID_BROADCOM, 0x0007, 0 },              /* BCM95701A7 */
11586         { PCI_VENDOR_ID_BROADCOM, 0x0008, PHY_ID_BCM5701 }, /* BCM95701A10 */
11587         { PCI_VENDOR_ID_BROADCOM, 0x8008, PHY_ID_BCM5701 }, /* BCM95701A12 */
11588         { PCI_VENDOR_ID_BROADCOM, 0x0009, PHY_ID_BCM5703 }, /* BCM95703Ax1 */
11589         { PCI_VENDOR_ID_BROADCOM, 0x8009, PHY_ID_BCM5703 }, /* BCM95703Ax2 */
11590
11591         /* 3com boards. */
11592         { PCI_VENDOR_ID_3COM, 0x1000, PHY_ID_BCM5401 }, /* 3C996T */
11593         { PCI_VENDOR_ID_3COM, 0x1006, PHY_ID_BCM5701 }, /* 3C996BT */
11594         { PCI_VENDOR_ID_3COM, 0x1004, 0 },              /* 3C996SX */
11595         { PCI_VENDOR_ID_3COM, 0x1007, PHY_ID_BCM5701 }, /* 3C1000T */
11596         { PCI_VENDOR_ID_3COM, 0x1008, PHY_ID_BCM5701 }, /* 3C940BR01 */
11597
11598         /* DELL boards. */
11599         { PCI_VENDOR_ID_DELL, 0x00d1, PHY_ID_BCM5401 }, /* VIPER */
11600         { PCI_VENDOR_ID_DELL, 0x0106, PHY_ID_BCM5401 }, /* JAGUAR */
11601         { PCI_VENDOR_ID_DELL, 0x0109, PHY_ID_BCM5411 }, /* MERLOT */
11602         { PCI_VENDOR_ID_DELL, 0x010a, PHY_ID_BCM5411 }, /* SLIM_MERLOT */
11603
11604         /* Compaq boards. */
11605         { PCI_VENDOR_ID_COMPAQ, 0x007c, PHY_ID_BCM5701 }, /* BANSHEE */
11606         { PCI_VENDOR_ID_COMPAQ, 0x009a, PHY_ID_BCM5701 }, /* BANSHEE_2 */
11607         { PCI_VENDOR_ID_COMPAQ, 0x007d, 0 },              /* CHANGELING */
11608         { PCI_VENDOR_ID_COMPAQ, 0x0085, PHY_ID_BCM5701 }, /* NC7780 */
11609         { PCI_VENDOR_ID_COMPAQ, 0x0099, PHY_ID_BCM5701 }, /* NC7780_2 */
11610
11611         /* IBM boards. */
11612         { PCI_VENDOR_ID_IBM, 0x0281, 0 } /* IBM??? */
11613 };
11614
11615 static inline struct subsys_tbl_ent *lookup_by_subsys(struct tg3 *tp)
11616 {
11617         int i;
11618
11619         for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
11620                 if ((subsys_id_to_phy_id[i].subsys_vendor ==
11621                      tp->pdev->subsystem_vendor) &&
11622                     (subsys_id_to_phy_id[i].subsys_devid ==
11623                      tp->pdev->subsystem_device))
11624                         return &subsys_id_to_phy_id[i];
11625         }
11626         return NULL;
11627 }
11628
11629 static void __devinit tg3_get_eeprom_hw_cfg(struct tg3 *tp)
11630 {
11631         u32 val;
11632         u16 pmcsr;
11633
11634         /* On some early chips the SRAM cannot be accessed in D3hot state,
11635          * so need make sure we're in D0.
11636          */
11637         pci_read_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, &pmcsr);
11638         pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
11639         pci_write_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, pmcsr);
11640         msleep(1);
11641
11642         /* Make sure register accesses (indirect or otherwise)
11643          * will function correctly.
11644          */
11645         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
11646                                tp->misc_host_ctrl);
11647
11648         /* The memory arbiter has to be enabled in order for SRAM accesses
11649          * to succeed.  Normally on powerup the tg3 chip firmware will make
11650          * sure it is enabled, but other entities such as system netboot
11651          * code might disable it.
11652          */
11653         val = tr32(MEMARB_MODE);
11654         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
11655
11656         tp->phy_id = PHY_ID_INVALID;
11657         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
11658
11659         /* Assume an onboard device and WOL capable by default.  */
11660         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT | TG3_FLAG_WOL_CAP;
11661
11662         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
11663                 if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
11664                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
11665                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
11666                 }
11667                 val = tr32(VCPU_CFGSHDW);
11668                 if (val & VCPU_CFGSHDW_ASPM_DBNC)
11669                         tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
11670                 if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
11671                     (val & VCPU_CFGSHDW_WOL_MAGPKT))
11672                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
11673                 goto done;
11674         }
11675
11676         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
11677         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
11678                 u32 nic_cfg, led_cfg;
11679                 u32 nic_phy_id, ver, cfg2 = 0, cfg4 = 0, eeprom_phy_id;
11680                 int eeprom_phy_serdes = 0;
11681
11682                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
11683                 tp->nic_sram_data_cfg = nic_cfg;
11684
11685                 tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
11686                 ver >>= NIC_SRAM_DATA_VER_SHIFT;
11687                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700) &&
11688                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) &&
11689                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5703) &&
11690                     (ver > 0) && (ver < 0x100))
11691                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
11692
11693                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
11694                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_4, &cfg4);
11695
11696                 if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
11697                     NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
11698                         eeprom_phy_serdes = 1;
11699
11700                 tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
11701                 if (nic_phy_id != 0) {
11702                         u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
11703                         u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
11704
11705                         eeprom_phy_id  = (id1 >> 16) << 10;
11706                         eeprom_phy_id |= (id2 & 0xfc00) << 16;
11707                         eeprom_phy_id |= (id2 & 0x03ff) <<  0;
11708                 } else
11709                         eeprom_phy_id = 0;
11710
11711                 tp->phy_id = eeprom_phy_id;
11712                 if (eeprom_phy_serdes) {
11713                         if (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)
11714                                 tp->tg3_flags2 |= TG3_FLG2_MII_SERDES;
11715                         else
11716                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
11717                 }
11718
11719                 if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
11720                         led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
11721                                     SHASTA_EXT_LED_MODE_MASK);
11722                 else
11723                         led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
11724
11725                 switch (led_cfg) {
11726                 default:
11727                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
11728                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
11729                         break;
11730
11731                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
11732                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
11733                         break;
11734
11735                 case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
11736                         tp->led_ctrl = LED_CTRL_MODE_MAC;
11737
11738                         /* Default to PHY_1_MODE if 0 (MAC_MODE) is
11739                          * read on some older 5700/5701 bootcode.
11740                          */
11741                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
11742                             ASIC_REV_5700 ||
11743                             GET_ASIC_REV(tp->pci_chip_rev_id) ==
11744                             ASIC_REV_5701)
11745                                 tp->led_ctrl = LED_CTRL_MODE_PHY_1;
11746
11747                         break;
11748
11749                 case SHASTA_EXT_LED_SHARED:
11750                         tp->led_ctrl = LED_CTRL_MODE_SHARED;
11751                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
11752                             tp->pci_chip_rev_id != CHIPREV_ID_5750_A1)
11753                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
11754                                                  LED_CTRL_MODE_PHY_2);
11755                         break;
11756
11757                 case SHASTA_EXT_LED_MAC:
11758                         tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
11759                         break;
11760
11761                 case SHASTA_EXT_LED_COMBO:
11762                         tp->led_ctrl = LED_CTRL_MODE_COMBO;
11763                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0)
11764                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
11765                                                  LED_CTRL_MODE_PHY_2);
11766                         break;
11767
11768                 }
11769
11770                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
11771                      GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) &&
11772                     tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
11773                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
11774
11775                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX)
11776                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
11777
11778                 if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
11779                         tp->tg3_flags |= TG3_FLAG_EEPROM_WRITE_PROT;
11780                         if ((tp->pdev->subsystem_vendor ==
11781                              PCI_VENDOR_ID_ARIMA) &&
11782                             (tp->pdev->subsystem_device == 0x205a ||
11783                              tp->pdev->subsystem_device == 0x2063))
11784                                 tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
11785                 } else {
11786                         tp->tg3_flags &= ~TG3_FLAG_EEPROM_WRITE_PROT;
11787                         tp->tg3_flags2 |= TG3_FLG2_IS_NIC;
11788                 }
11789
11790                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
11791                         tp->tg3_flags |= TG3_FLAG_ENABLE_ASF;
11792                         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS)
11793                                 tp->tg3_flags2 |= TG3_FLG2_ASF_NEW_HANDSHAKE;
11794                 }
11795
11796                 if ((nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE) &&
11797                         (tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
11798                         tp->tg3_flags3 |= TG3_FLG3_ENABLE_APE;
11799
11800                 if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES &&
11801                     !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
11802                         tp->tg3_flags &= ~TG3_FLAG_WOL_CAP;
11803
11804                 if ((tp->tg3_flags & TG3_FLAG_WOL_CAP) &&
11805                     (nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE))
11806                         tp->tg3_flags |= TG3_FLAG_WOL_ENABLE;
11807
11808                 if (cfg2 & (1 << 17))
11809                         tp->tg3_flags2 |= TG3_FLG2_CAPACITIVE_COUPLING;
11810
11811                 /* serdes signal pre-emphasis in register 0x590 set by */
11812                 /* bootcode if bit 18 is set */
11813                 if (cfg2 & (1 << 18))
11814                         tp->tg3_flags2 |= TG3_FLG2_SERDES_PREEMPHASIS;
11815
11816                 if (((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
11817                       GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX)) &&
11818                     (cfg2 & NIC_SRAM_DATA_CFG_2_APD_EN))
11819                         tp->tg3_flags3 |= TG3_FLG3_PHY_ENABLE_APD;
11820
11821                 if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
11822                         u32 cfg3;
11823
11824                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
11825                         if (cfg3 & NIC_SRAM_ASPM_DEBOUNCE)
11826                                 tp->tg3_flags |= TG3_FLAG_ASPM_WORKAROUND;
11827                 }
11828
11829                 if (cfg4 & NIC_SRAM_RGMII_STD_IBND_DISABLE)
11830                         tp->tg3_flags3 |= TG3_FLG3_RGMII_STD_IBND_DISABLE;
11831                 if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_RX_EN)
11832                         tp->tg3_flags3 |= TG3_FLG3_RGMII_EXT_IBND_RX_EN;
11833                 if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_TX_EN)
11834                         tp->tg3_flags3 |= TG3_FLG3_RGMII_EXT_IBND_TX_EN;
11835         }
11836 done:
11837         device_init_wakeup(&tp->pdev->dev, tp->tg3_flags & TG3_FLAG_WOL_CAP);
11838         device_set_wakeup_enable(&tp->pdev->dev,
11839                                  tp->tg3_flags & TG3_FLAG_WOL_ENABLE);
11840 }
11841
11842 static int __devinit tg3_issue_otp_command(struct tg3 *tp, u32 cmd)
11843 {
11844         int i;
11845         u32 val;
11846
11847         tw32(OTP_CTRL, cmd | OTP_CTRL_OTP_CMD_START);
11848         tw32(OTP_CTRL, cmd);
11849
11850         /* Wait for up to 1 ms for command to execute. */
11851         for (i = 0; i < 100; i++) {
11852                 val = tr32(OTP_STATUS);
11853                 if (val & OTP_STATUS_CMD_DONE)
11854                         break;
11855                 udelay(10);
11856         }
11857
11858         return (val & OTP_STATUS_CMD_DONE) ? 0 : -EBUSY;
11859 }
11860
11861 /* Read the gphy configuration from the OTP region of the chip.  The gphy
11862  * configuration is a 32-bit value that straddles the alignment boundary.
11863  * We do two 32-bit reads and then shift and merge the results.
11864  */
11865 static u32 __devinit tg3_read_otp_phycfg(struct tg3 *tp)
11866 {
11867         u32 bhalf_otp, thalf_otp;
11868
11869         tw32(OTP_MODE, OTP_MODE_OTP_THRU_GRC);
11870
11871         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_INIT))
11872                 return 0;
11873
11874         tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC1);
11875
11876         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
11877                 return 0;
11878
11879         thalf_otp = tr32(OTP_READ_DATA);
11880
11881         tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC2);
11882
11883         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
11884                 return 0;
11885
11886         bhalf_otp = tr32(OTP_READ_DATA);
11887
11888         return ((thalf_otp & 0x0000ffff) << 16) | (bhalf_otp >> 16);
11889 }
11890
11891 static int __devinit tg3_phy_probe(struct tg3 *tp)
11892 {
11893         u32 hw_phy_id_1, hw_phy_id_2;
11894         u32 hw_phy_id, hw_phy_id_masked;
11895         int err;
11896
11897         if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB)
11898                 return tg3_phy_init(tp);
11899
11900         /* Reading the PHY ID register can conflict with ASF
11901          * firmware access to the PHY hardware.
11902          */
11903         err = 0;
11904         if ((tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
11905             (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)) {
11906                 hw_phy_id = hw_phy_id_masked = PHY_ID_INVALID;
11907         } else {
11908                 /* Now read the physical PHY_ID from the chip and verify
11909                  * that it is sane.  If it doesn't look good, we fall back
11910                  * to either the hard-coded table based PHY_ID and failing
11911                  * that the value found in the eeprom area.
11912                  */
11913                 err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
11914                 err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
11915
11916                 hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
11917                 hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
11918                 hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
11919
11920                 hw_phy_id_masked = hw_phy_id & PHY_ID_MASK;
11921         }
11922
11923         if (!err && KNOWN_PHY_ID(hw_phy_id_masked)) {
11924                 tp->phy_id = hw_phy_id;
11925                 if (hw_phy_id_masked == PHY_ID_BCM8002)
11926                         tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
11927                 else
11928                         tp->tg3_flags2 &= ~TG3_FLG2_PHY_SERDES;
11929         } else {
11930                 if (tp->phy_id != PHY_ID_INVALID) {
11931                         /* Do nothing, phy ID already set up in
11932                          * tg3_get_eeprom_hw_cfg().
11933                          */
11934                 } else {
11935                         struct subsys_tbl_ent *p;
11936
11937                         /* No eeprom signature?  Try the hardcoded
11938                          * subsys device table.
11939                          */
11940                         p = lookup_by_subsys(tp);
11941                         if (!p)
11942                                 return -ENODEV;
11943
11944                         tp->phy_id = p->phy_id;
11945                         if (!tp->phy_id ||
11946                             tp->phy_id == PHY_ID_BCM8002)
11947                                 tp->tg3_flags2 |= TG3_FLG2_PHY_SERDES;
11948                 }
11949         }
11950
11951         if (!(tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) &&
11952             !(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) &&
11953             !(tp->tg3_flags & TG3_FLAG_ENABLE_ASF)) {
11954                 u32 bmsr, adv_reg, tg3_ctrl, mask;
11955
11956                 tg3_readphy(tp, MII_BMSR, &bmsr);
11957                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
11958                     (bmsr & BMSR_LSTATUS))
11959                         goto skip_phy_reset;
11960
11961                 err = tg3_phy_reset(tp);
11962                 if (err)
11963                         return err;
11964
11965                 adv_reg = (ADVERTISE_10HALF | ADVERTISE_10FULL |
11966                            ADVERTISE_100HALF | ADVERTISE_100FULL |
11967                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
11968                 tg3_ctrl = 0;
11969                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY)) {
11970                         tg3_ctrl = (MII_TG3_CTRL_ADV_1000_HALF |
11971                                     MII_TG3_CTRL_ADV_1000_FULL);
11972                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
11973                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
11974                                 tg3_ctrl |= (MII_TG3_CTRL_AS_MASTER |
11975                                              MII_TG3_CTRL_ENABLE_AS_MASTER);
11976                 }
11977
11978                 mask = (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
11979                         ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
11980                         ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full);
11981                 if (!tg3_copper_is_advertising_all(tp, mask)) {
11982                         tg3_writephy(tp, MII_ADVERTISE, adv_reg);
11983
11984                         if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
11985                                 tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
11986
11987                         tg3_writephy(tp, MII_BMCR,
11988                                      BMCR_ANENABLE | BMCR_ANRESTART);
11989                 }
11990                 tg3_phy_set_wirespeed(tp);
11991
11992                 tg3_writephy(tp, MII_ADVERTISE, adv_reg);
11993                 if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY))
11994                         tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
11995         }
11996
11997 skip_phy_reset:
11998         if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
11999                 err = tg3_init_5401phy_dsp(tp);
12000                 if (err)
12001                         return err;
12002         }
12003
12004         if (!err && ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401)) {
12005                 err = tg3_init_5401phy_dsp(tp);
12006         }
12007
12008         if (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES)
12009                 tp->link_config.advertising =
12010                         (ADVERTISED_1000baseT_Half |
12011                          ADVERTISED_1000baseT_Full |
12012                          ADVERTISED_Autoneg |
12013                          ADVERTISED_FIBRE);
12014         if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
12015                 tp->link_config.advertising &=
12016                         ~(ADVERTISED_1000baseT_Half |
12017                           ADVERTISED_1000baseT_Full);
12018
12019         return err;
12020 }
12021
12022 static void __devinit tg3_read_partno(struct tg3 *tp)
12023 {
12024         unsigned char vpd_data[256];   /* in little-endian format */
12025         unsigned int i;
12026         u32 magic;
12027
12028         if ((tp->tg3_flags3 & TG3_FLG3_NO_NVRAM) ||
12029             tg3_nvram_read(tp, 0x0, &magic))
12030                 goto out_not_found;
12031
12032         if (magic == TG3_EEPROM_MAGIC) {
12033                 for (i = 0; i < 256; i += 4) {
12034                         u32 tmp;
12035
12036                         /* The data is in little-endian format in NVRAM.
12037                          * Use the big-endian read routines to preserve
12038                          * the byte order as it exists in NVRAM.
12039                          */
12040                         if (tg3_nvram_read_be32(tp, 0x100 + i, &tmp))
12041                                 goto out_not_found;
12042
12043                         memcpy(&vpd_data[i], &tmp, sizeof(tmp));
12044                 }
12045         } else {
12046                 int vpd_cap;
12047
12048                 vpd_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_VPD);
12049                 for (i = 0; i < 256; i += 4) {
12050                         u32 tmp, j = 0;
12051                         __le32 v;
12052                         u16 tmp16;
12053
12054                         pci_write_config_word(tp->pdev, vpd_cap + PCI_VPD_ADDR,
12055                                               i);
12056                         while (j++ < 100) {
12057                                 pci_read_config_word(tp->pdev, vpd_cap +
12058                                                      PCI_VPD_ADDR, &tmp16);
12059                                 if (tmp16 & 0x8000)
12060                                         break;
12061                                 msleep(1);
12062                         }
12063                         if (!(tmp16 & 0x8000))
12064                                 goto out_not_found;
12065
12066                         pci_read_config_dword(tp->pdev, vpd_cap + PCI_VPD_DATA,
12067                                               &tmp);
12068                         v = cpu_to_le32(tmp);
12069                         memcpy(&vpd_data[i], &v, sizeof(v));
12070                 }
12071         }
12072
12073         /* Now parse and find the part number. */
12074         for (i = 0; i < 254; ) {
12075                 unsigned char val = vpd_data[i];
12076                 unsigned int block_end;
12077
12078                 if (val == 0x82 || val == 0x91) {
12079                         i = (i + 3 +
12080                              (vpd_data[i + 1] +
12081                               (vpd_data[i + 2] << 8)));
12082                         continue;
12083                 }
12084
12085                 if (val != 0x90)
12086                         goto out_not_found;
12087
12088                 block_end = (i + 3 +
12089                              (vpd_data[i + 1] +
12090                               (vpd_data[i + 2] << 8)));
12091                 i += 3;
12092
12093                 if (block_end > 256)
12094                         goto out_not_found;
12095
12096                 while (i < (block_end - 2)) {
12097                         if (vpd_data[i + 0] == 'P' &&
12098                             vpd_data[i + 1] == 'N') {
12099                                 int partno_len = vpd_data[i + 2];
12100
12101                                 i += 3;
12102                                 if (partno_len > 24 || (partno_len + i) > 256)
12103                                         goto out_not_found;
12104
12105                                 memcpy(tp->board_part_number,
12106                                        &vpd_data[i], partno_len);
12107
12108                                 /* Success. */
12109                                 return;
12110                         }
12111                         i += 3 + vpd_data[i + 2];
12112                 }
12113
12114                 /* Part number not found. */
12115                 goto out_not_found;
12116         }
12117
12118 out_not_found:
12119         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
12120                 strcpy(tp->board_part_number, "BCM95906");
12121         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 &&
12122                  tp->pdev->device == TG3PCI_DEVICE_TIGON3_57780)
12123                 strcpy(tp->board_part_number, "BCM57780");
12124         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 &&
12125                  tp->pdev->device == TG3PCI_DEVICE_TIGON3_57760)
12126                 strcpy(tp->board_part_number, "BCM57760");
12127         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 &&
12128                  tp->pdev->device == TG3PCI_DEVICE_TIGON3_57790)
12129                 strcpy(tp->board_part_number, "BCM57790");
12130         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 &&
12131                  tp->pdev->device == TG3PCI_DEVICE_TIGON3_57788)
12132                 strcpy(tp->board_part_number, "BCM57788");
12133         else
12134                 strcpy(tp->board_part_number, "none");
12135 }
12136
12137 static int __devinit tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
12138 {
12139         u32 val;
12140
12141         if (tg3_nvram_read(tp, offset, &val) ||
12142             (val & 0xfc000000) != 0x0c000000 ||
12143             tg3_nvram_read(tp, offset + 4, &val) ||
12144             val != 0)
12145                 return 0;
12146
12147         return 1;
12148 }
12149
12150 static void __devinit tg3_read_bc_ver(struct tg3 *tp)
12151 {
12152         u32 val, offset, start, ver_offset;
12153         int i;
12154         bool newver = false;
12155
12156         if (tg3_nvram_read(tp, 0xc, &offset) ||
12157             tg3_nvram_read(tp, 0x4, &start))
12158                 return;
12159
12160         offset = tg3_nvram_logical_addr(tp, offset);
12161
12162         if (tg3_nvram_read(tp, offset, &val))
12163                 return;
12164
12165         if ((val & 0xfc000000) == 0x0c000000) {
12166                 if (tg3_nvram_read(tp, offset + 4, &val))
12167                         return;
12168
12169                 if (val == 0)
12170                         newver = true;
12171         }
12172
12173         if (newver) {
12174                 if (tg3_nvram_read(tp, offset + 8, &ver_offset))
12175                         return;
12176
12177                 offset = offset + ver_offset - start;
12178                 for (i = 0; i < 16; i += 4) {
12179                         __be32 v;
12180                         if (tg3_nvram_read_be32(tp, offset + i, &v))
12181                                 return;
12182
12183                         memcpy(tp->fw_ver + i, &v, sizeof(v));
12184                 }
12185         } else {
12186                 u32 major, minor;
12187
12188                 if (tg3_nvram_read(tp, TG3_NVM_PTREV_BCVER, &ver_offset))
12189                         return;
12190
12191                 major = (ver_offset & TG3_NVM_BCVER_MAJMSK) >>
12192                         TG3_NVM_BCVER_MAJSFT;
12193                 minor = ver_offset & TG3_NVM_BCVER_MINMSK;
12194                 snprintf(&tp->fw_ver[0], 32, "v%d.%02d", major, minor);
12195         }
12196 }
12197
12198 static void __devinit tg3_read_hwsb_ver(struct tg3 *tp)
12199 {
12200         u32 val, major, minor;
12201
12202         /* Use native endian representation */
12203         if (tg3_nvram_read(tp, TG3_NVM_HWSB_CFG1, &val))
12204                 return;
12205
12206         major = (val & TG3_NVM_HWSB_CFG1_MAJMSK) >>
12207                 TG3_NVM_HWSB_CFG1_MAJSFT;
12208         minor = (val & TG3_NVM_HWSB_CFG1_MINMSK) >>
12209                 TG3_NVM_HWSB_CFG1_MINSFT;
12210
12211         snprintf(&tp->fw_ver[0], 32, "sb v%d.%02d", major, minor);
12212 }
12213
12214 static void __devinit tg3_read_sb_ver(struct tg3 *tp, u32 val)
12215 {
12216         u32 offset, major, minor, build;
12217
12218         tp->fw_ver[0] = 's';
12219         tp->fw_ver[1] = 'b';
12220         tp->fw_ver[2] = '\0';
12221
12222         if ((val & TG3_EEPROM_SB_FORMAT_MASK) != TG3_EEPROM_SB_FORMAT_1)
12223                 return;
12224
12225         switch (val & TG3_EEPROM_SB_REVISION_MASK) {
12226         case TG3_EEPROM_SB_REVISION_0:
12227                 offset = TG3_EEPROM_SB_F1R0_EDH_OFF;
12228                 break;
12229         case TG3_EEPROM_SB_REVISION_2:
12230                 offset = TG3_EEPROM_SB_F1R2_EDH_OFF;
12231                 break;
12232         case TG3_EEPROM_SB_REVISION_3:
12233                 offset = TG3_EEPROM_SB_F1R3_EDH_OFF;
12234                 break;
12235         default:
12236                 return;
12237         }
12238
12239         if (tg3_nvram_read(tp, offset, &val))
12240                 return;
12241
12242         build = (val & TG3_EEPROM_SB_EDH_BLD_MASK) >>
12243                 TG3_EEPROM_SB_EDH_BLD_SHFT;
12244         major = (val & TG3_EEPROM_SB_EDH_MAJ_MASK) >>
12245                 TG3_EEPROM_SB_EDH_MAJ_SHFT;
12246         minor =  val & TG3_EEPROM_SB_EDH_MIN_MASK;
12247
12248         if (minor > 99 || build > 26)
12249                 return;
12250
12251         snprintf(&tp->fw_ver[2], 30, " v%d.%02d", major, minor);
12252
12253         if (build > 0) {
12254                 tp->fw_ver[8] = 'a' + build - 1;
12255                 tp->fw_ver[9] = '\0';
12256         }
12257 }
12258
12259 static void __devinit tg3_read_mgmtfw_ver(struct tg3 *tp)
12260 {
12261         u32 val, offset, start;
12262         int i, vlen;
12263
12264         for (offset = TG3_NVM_DIR_START;
12265              offset < TG3_NVM_DIR_END;
12266              offset += TG3_NVM_DIRENT_SIZE) {
12267                 if (tg3_nvram_read(tp, offset, &val))
12268                         return;
12269
12270                 if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
12271                         break;
12272         }
12273
12274         if (offset == TG3_NVM_DIR_END)
12275                 return;
12276
12277         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS))
12278                 start = 0x08000000;
12279         else if (tg3_nvram_read(tp, offset - 4, &start))
12280                 return;
12281
12282         if (tg3_nvram_read(tp, offset + 4, &offset) ||
12283             !tg3_fw_img_is_valid(tp, offset) ||
12284             tg3_nvram_read(tp, offset + 8, &val))
12285                 return;
12286
12287         offset += val - start;
12288
12289         vlen = strlen(tp->fw_ver);
12290
12291         tp->fw_ver[vlen++] = ',';
12292         tp->fw_ver[vlen++] = ' ';
12293
12294         for (i = 0; i < 4; i++) {
12295                 __be32 v;
12296                 if (tg3_nvram_read_be32(tp, offset, &v))
12297                         return;
12298
12299                 offset += sizeof(v);
12300
12301                 if (vlen > TG3_VER_SIZE - sizeof(v)) {
12302                         memcpy(&tp->fw_ver[vlen], &v, TG3_VER_SIZE - vlen);
12303                         break;
12304                 }
12305
12306                 memcpy(&tp->fw_ver[vlen], &v, sizeof(v));
12307                 vlen += sizeof(v);
12308         }
12309 }
12310
12311 static void __devinit tg3_read_dash_ver(struct tg3 *tp)
12312 {
12313         int vlen;
12314         u32 apedata;
12315
12316         if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) ||
12317             !(tp->tg3_flags  & TG3_FLAG_ENABLE_ASF))
12318                 return;
12319
12320         apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
12321         if (apedata != APE_SEG_SIG_MAGIC)
12322                 return;
12323
12324         apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
12325         if (!(apedata & APE_FW_STATUS_READY))
12326                 return;
12327
12328         apedata = tg3_ape_read32(tp, TG3_APE_FW_VERSION);
12329
12330         vlen = strlen(tp->fw_ver);
12331
12332         snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " DASH v%d.%d.%d.%d",
12333                  (apedata & APE_FW_VERSION_MAJMSK) >> APE_FW_VERSION_MAJSFT,
12334                  (apedata & APE_FW_VERSION_MINMSK) >> APE_FW_VERSION_MINSFT,
12335                  (apedata & APE_FW_VERSION_REVMSK) >> APE_FW_VERSION_REVSFT,
12336                  (apedata & APE_FW_VERSION_BLDMSK));
12337 }
12338
12339 static void __devinit tg3_read_fw_ver(struct tg3 *tp)
12340 {
12341         u32 val;
12342
12343         if (tp->tg3_flags3 & TG3_FLG3_NO_NVRAM) {
12344                 tp->fw_ver[0] = 's';
12345                 tp->fw_ver[1] = 'b';
12346                 tp->fw_ver[2] = '\0';
12347
12348                 return;
12349         }
12350
12351         if (tg3_nvram_read(tp, 0, &val))
12352                 return;
12353
12354         if (val == TG3_EEPROM_MAGIC)
12355                 tg3_read_bc_ver(tp);
12356         else if ((val & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW)
12357                 tg3_read_sb_ver(tp, val);
12358         else if ((val & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
12359                 tg3_read_hwsb_ver(tp);
12360         else
12361                 return;
12362
12363         if (!(tp->tg3_flags & TG3_FLAG_ENABLE_ASF) ||
12364              (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE))
12365                 return;
12366
12367         tg3_read_mgmtfw_ver(tp);
12368
12369         tp->fw_ver[TG3_VER_SIZE - 1] = 0;
12370 }
12371
12372 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *);
12373
12374 static int __devinit tg3_get_invariants(struct tg3 *tp)
12375 {
12376         static struct pci_device_id write_reorder_chipsets[] = {
12377                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
12378                              PCI_DEVICE_ID_AMD_FE_GATE_700C) },
12379                 { PCI_DEVICE(PCI_VENDOR_ID_AMD,
12380                              PCI_DEVICE_ID_AMD_8131_BRIDGE) },
12381                 { PCI_DEVICE(PCI_VENDOR_ID_VIA,
12382                              PCI_DEVICE_ID_VIA_8385_0) },
12383                 { },
12384         };
12385         u32 misc_ctrl_reg;
12386         u32 pci_state_reg, grc_misc_cfg;
12387         u32 val;
12388         u16 pci_cmd;
12389         int err;
12390
12391         /* Force memory write invalidate off.  If we leave it on,
12392          * then on 5700_BX chips we have to enable a workaround.
12393          * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
12394          * to match the cacheline size.  The Broadcom driver have this
12395          * workaround but turns MWI off all the times so never uses
12396          * it.  This seems to suggest that the workaround is insufficient.
12397          */
12398         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
12399         pci_cmd &= ~PCI_COMMAND_INVALIDATE;
12400         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
12401
12402         /* It is absolutely critical that TG3PCI_MISC_HOST_CTRL
12403          * has the register indirect write enable bit set before
12404          * we try to access any of the MMIO registers.  It is also
12405          * critical that the PCI-X hw workaround situation is decided
12406          * before that as well.
12407          */
12408         pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
12409                               &misc_ctrl_reg);
12410
12411         tp->pci_chip_rev_id = (misc_ctrl_reg >>
12412                                MISC_HOST_CTRL_CHIPREV_SHIFT);
12413         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_USE_PROD_ID_REG) {
12414                 u32 prod_id_asic_rev;
12415
12416                 if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717C ||
12417                     tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717S ||
12418                     tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718C ||
12419                     tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718S)
12420                         pci_read_config_dword(tp->pdev,
12421                                               TG3PCI_GEN2_PRODID_ASICREV,
12422                                               &prod_id_asic_rev);
12423                 else
12424                         pci_read_config_dword(tp->pdev, TG3PCI_PRODID_ASICREV,
12425                                               &prod_id_asic_rev);
12426
12427                 tp->pci_chip_rev_id = prod_id_asic_rev;
12428         }
12429
12430         /* Wrong chip ID in 5752 A0. This code can be removed later
12431          * as A0 is not in production.
12432          */
12433         if (tp->pci_chip_rev_id == CHIPREV_ID_5752_A0_HW)
12434                 tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
12435
12436         /* If we have 5702/03 A1 or A2 on certain ICH chipsets,
12437          * we need to disable memory and use config. cycles
12438          * only to access all registers. The 5702/03 chips
12439          * can mistakenly decode the special cycles from the
12440          * ICH chipsets as memory write cycles, causing corruption
12441          * of register and memory space. Only certain ICH bridges
12442          * will drive special cycles with non-zero data during the
12443          * address phase which can fall within the 5703's address
12444          * range. This is not an ICH bug as the PCI spec allows
12445          * non-zero address during special cycles. However, only
12446          * these ICH bridges are known to drive non-zero addresses
12447          * during special cycles.
12448          *
12449          * Since special cycles do not cross PCI bridges, we only
12450          * enable this workaround if the 5703 is on the secondary
12451          * bus of these ICH bridges.
12452          */
12453         if ((tp->pci_chip_rev_id == CHIPREV_ID_5703_A1) ||
12454             (tp->pci_chip_rev_id == CHIPREV_ID_5703_A2)) {
12455                 static struct tg3_dev_id {
12456                         u32     vendor;
12457                         u32     device;
12458                         u32     rev;
12459                 } ich_chipsets[] = {
12460                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
12461                           PCI_ANY_ID },
12462                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
12463                           PCI_ANY_ID },
12464                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
12465                           0xa },
12466                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
12467                           PCI_ANY_ID },
12468                         { },
12469                 };
12470                 struct tg3_dev_id *pci_id = &ich_chipsets[0];
12471                 struct pci_dev *bridge = NULL;
12472
12473                 while (pci_id->vendor != 0) {
12474                         bridge = pci_get_device(pci_id->vendor, pci_id->device,
12475                                                 bridge);
12476                         if (!bridge) {
12477                                 pci_id++;
12478                                 continue;
12479                         }
12480                         if (pci_id->rev != PCI_ANY_ID) {
12481                                 if (bridge->revision > pci_id->rev)
12482                                         continue;
12483                         }
12484                         if (bridge->subordinate &&
12485                             (bridge->subordinate->number ==
12486                              tp->pdev->bus->number)) {
12487
12488                                 tp->tg3_flags2 |= TG3_FLG2_ICH_WORKAROUND;
12489                                 pci_dev_put(bridge);
12490                                 break;
12491                         }
12492                 }
12493         }
12494
12495         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
12496                 static struct tg3_dev_id {
12497                         u32     vendor;
12498                         u32     device;
12499                 } bridge_chipsets[] = {
12500                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_0 },
12501                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_1 },
12502                         { },
12503                 };
12504                 struct tg3_dev_id *pci_id = &bridge_chipsets[0];
12505                 struct pci_dev *bridge = NULL;
12506
12507                 while (pci_id->vendor != 0) {
12508                         bridge = pci_get_device(pci_id->vendor,
12509                                                 pci_id->device,
12510                                                 bridge);
12511                         if (!bridge) {
12512                                 pci_id++;
12513                                 continue;
12514                         }
12515                         if (bridge->subordinate &&
12516                             (bridge->subordinate->number <=
12517                              tp->pdev->bus->number) &&
12518                             (bridge->subordinate->subordinate >=
12519                              tp->pdev->bus->number)) {
12520                                 tp->tg3_flags3 |= TG3_FLG3_5701_DMA_BUG;
12521                                 pci_dev_put(bridge);
12522                                 break;
12523                         }
12524                 }
12525         }
12526
12527         /* The EPB bridge inside 5714, 5715, and 5780 cannot support
12528          * DMA addresses > 40-bit. This bridge may have other additional
12529          * 57xx devices behind it in some 4-port NIC designs for example.
12530          * Any tg3 device found behind the bridge will also need the 40-bit
12531          * DMA workaround.
12532          */
12533         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
12534             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
12535                 tp->tg3_flags2 |= TG3_FLG2_5780_CLASS;
12536                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
12537                 tp->msi_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_MSI);
12538         }
12539         else {
12540                 struct pci_dev *bridge = NULL;
12541
12542                 do {
12543                         bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
12544                                                 PCI_DEVICE_ID_SERVERWORKS_EPB,
12545                                                 bridge);
12546                         if (bridge && bridge->subordinate &&
12547                             (bridge->subordinate->number <=
12548                              tp->pdev->bus->number) &&
12549                             (bridge->subordinate->subordinate >=
12550                              tp->pdev->bus->number)) {
12551                                 tp->tg3_flags |= TG3_FLAG_40BIT_DMA_BUG;
12552                                 pci_dev_put(bridge);
12553                                 break;
12554                         }
12555                 } while (bridge);
12556         }
12557
12558         /* Initialize misc host control in PCI block. */
12559         tp->misc_host_ctrl |= (misc_ctrl_reg &
12560                                MISC_HOST_CTRL_CHIPREV);
12561         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
12562                                tp->misc_host_ctrl);
12563
12564         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
12565             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 ||
12566             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
12567                 tp->pdev_peer = tg3_find_peer(tp);
12568
12569         /* Intentionally exclude ASIC_REV_5906 */
12570         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
12571             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
12572             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12573             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
12574             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
12575             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
12576             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
12577                 tp->tg3_flags3 |= TG3_FLG3_5755_PLUS;
12578
12579         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
12580             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
12581             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
12582             (tp->tg3_flags3 & TG3_FLG3_5755_PLUS) ||
12583             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
12584                 tp->tg3_flags2 |= TG3_FLG2_5750_PLUS;
12585
12586         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) ||
12587             (tp->tg3_flags2 & TG3_FLG2_5750_PLUS))
12588                 tp->tg3_flags2 |= TG3_FLG2_5705_PLUS;
12589
12590         /* 5700 B0 chips do not support checksumming correctly due
12591          * to hardware bugs.
12592          */
12593         if (tp->pci_chip_rev_id == CHIPREV_ID_5700_B0)
12594                 tp->tg3_flags |= TG3_FLAG_BROKEN_CHECKSUMS;
12595         else {
12596                 tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS;
12597                 tp->dev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
12598                 if (tp->tg3_flags3 & TG3_FLG3_5755_PLUS)
12599                         tp->dev->features |= NETIF_F_IPV6_CSUM;
12600         }
12601
12602         if (tp->tg3_flags2 & TG3_FLG2_5750_PLUS) {
12603                 tp->tg3_flags |= TG3_FLAG_SUPPORT_MSI;
12604                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX ||
12605                     GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX ||
12606                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 &&
12607                      tp->pci_chip_rev_id <= CHIPREV_ID_5714_A2 &&
12608                      tp->pdev_peer == tp->pdev))
12609                         tp->tg3_flags &= ~TG3_FLAG_SUPPORT_MSI;
12610
12611                 if ((tp->tg3_flags3 & TG3_FLG3_5755_PLUS) ||
12612                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
12613                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_2;
12614                         tp->tg3_flags2 |= TG3_FLG2_1SHOT_MSI;
12615                 } else {
12616                         tp->tg3_flags2 |= TG3_FLG2_HW_TSO_1 | TG3_FLG2_TSO_BUG;
12617                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
12618                                 ASIC_REV_5750 &&
12619                             tp->pci_chip_rev_id >= CHIPREV_ID_5750_C2)
12620                                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_BUG;
12621                 }
12622         }
12623
12624         tp->irq_max = 1;
12625
12626 #ifdef TG3_NAPI
12627         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
12628                 tp->tg3_flags |= TG3_FLAG_SUPPORT_MSIX;
12629                 tp->irq_max = TG3_IRQ_MAX_VECS;
12630         }
12631 #endif
12632
12633         if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
12634              (tp->tg3_flags2 & TG3_FLG2_5780_CLASS) ||
12635             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
12636                 tp->tg3_flags |= TG3_FLAG_JUMBO_CAPABLE;
12637
12638         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
12639                               &pci_state_reg);
12640
12641         tp->pcie_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_EXP);
12642         if (tp->pcie_cap != 0) {
12643                 u16 lnkctl;
12644
12645                 tp->tg3_flags2 |= TG3_FLG2_PCI_EXPRESS;
12646
12647                 pcie_set_readrq(tp->pdev, 4096);
12648
12649                 pci_read_config_word(tp->pdev,
12650                                      tp->pcie_cap + PCI_EXP_LNKCTL,
12651                                      &lnkctl);
12652                 if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN) {
12653                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
12654                                 tp->tg3_flags2 &= ~TG3_FLG2_HW_TSO_2;
12655                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12656                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
12657                             tp->pci_chip_rev_id == CHIPREV_ID_57780_A0 ||
12658                             tp->pci_chip_rev_id == CHIPREV_ID_57780_A1)
12659                                 tp->tg3_flags3 |= TG3_FLG3_CLKREQ_BUG;
12660                 }
12661         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) {
12662                 tp->tg3_flags2 |= TG3_FLG2_PCI_EXPRESS;
12663         } else if (!(tp->tg3_flags2 & TG3_FLG2_5705_PLUS) ||
12664                    (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
12665                 tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
12666                 if (!tp->pcix_cap) {
12667                         printk(KERN_ERR PFX "Cannot find PCI-X "
12668                                             "capability, aborting.\n");
12669                         return -EIO;
12670                 }
12671
12672                 if (!(pci_state_reg & PCISTATE_CONV_PCI_MODE))
12673                         tp->tg3_flags |= TG3_FLAG_PCIX_MODE;
12674         }
12675
12676         /* If we have an AMD 762 or VIA K8T800 chipset, write
12677          * reordering to the mailbox registers done by the host
12678          * controller can cause major troubles.  We read back from
12679          * every mailbox register write to force the writes to be
12680          * posted to the chip in order.
12681          */
12682         if (pci_dev_present(write_reorder_chipsets) &&
12683             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
12684                 tp->tg3_flags |= TG3_FLAG_MBOX_WRITE_REORDER;
12685
12686         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
12687                              &tp->pci_cacheline_sz);
12688         pci_read_config_byte(tp->pdev, PCI_LATENCY_TIMER,
12689                              &tp->pci_lat_timer);
12690         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
12691             tp->pci_lat_timer < 64) {
12692                 tp->pci_lat_timer = 64;
12693                 pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
12694                                       tp->pci_lat_timer);
12695         }
12696
12697         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX) {
12698                 /* 5700 BX chips need to have their TX producer index
12699                  * mailboxes written twice to workaround a bug.
12700                  */
12701                 tp->tg3_flags |= TG3_FLAG_TXD_MBOX_HWBUG;
12702
12703                 /* If we are in PCI-X mode, enable register write workaround.
12704                  *
12705                  * The workaround is to use indirect register accesses
12706                  * for all chip writes not to mailbox registers.
12707                  */
12708                 if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
12709                         u32 pm_reg;
12710
12711                         tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
12712
12713                         /* The chip can have it's power management PCI config
12714                          * space registers clobbered due to this bug.
12715                          * So explicitly force the chip into D0 here.
12716                          */
12717                         pci_read_config_dword(tp->pdev,
12718                                               tp->pm_cap + PCI_PM_CTRL,
12719                                               &pm_reg);
12720                         pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
12721                         pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
12722                         pci_write_config_dword(tp->pdev,
12723                                                tp->pm_cap + PCI_PM_CTRL,
12724                                                pm_reg);
12725
12726                         /* Also, force SERR#/PERR# in PCI command. */
12727                         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
12728                         pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
12729                         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
12730                 }
12731         }
12732
12733         if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
12734                 tp->tg3_flags |= TG3_FLAG_PCI_HIGH_SPEED;
12735         if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
12736                 tp->tg3_flags |= TG3_FLAG_PCI_32BIT;
12737
12738         /* Chip-specific fixup from Broadcom driver */
12739         if ((tp->pci_chip_rev_id == CHIPREV_ID_5704_A0) &&
12740             (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
12741                 pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
12742                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
12743         }
12744
12745         /* Default fast path register access methods */
12746         tp->read32 = tg3_read32;
12747         tp->write32 = tg3_write32;
12748         tp->read32_mbox = tg3_read32;
12749         tp->write32_mbox = tg3_write32;
12750         tp->write32_tx_mbox = tg3_write32;
12751         tp->write32_rx_mbox = tg3_write32;
12752
12753         /* Various workaround register access methods */
12754         if (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG)
12755                 tp->write32 = tg3_write_indirect_reg32;
12756         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
12757                  ((tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) &&
12758                   tp->pci_chip_rev_id == CHIPREV_ID_5750_A0)) {
12759                 /*
12760                  * Back to back register writes can cause problems on these
12761                  * chips, the workaround is to read back all reg writes
12762                  * except those to mailbox regs.
12763                  *
12764                  * See tg3_write_indirect_reg32().
12765                  */
12766                 tp->write32 = tg3_write_flush_reg32;
12767         }
12768
12769         if ((tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG) ||
12770             (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)) {
12771                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
12772                 if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
12773                         tp->write32_rx_mbox = tg3_write_flush_reg32;
12774         }
12775
12776         if (tp->tg3_flags2 & TG3_FLG2_ICH_WORKAROUND) {
12777                 tp->read32 = tg3_read_indirect_reg32;
12778                 tp->write32 = tg3_write_indirect_reg32;
12779                 tp->read32_mbox = tg3_read_indirect_mbox;
12780                 tp->write32_mbox = tg3_write_indirect_mbox;
12781                 tp->write32_tx_mbox = tg3_write_indirect_mbox;
12782                 tp->write32_rx_mbox = tg3_write_indirect_mbox;
12783
12784                 iounmap(tp->regs);
12785                 tp->regs = NULL;
12786
12787                 pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
12788                 pci_cmd &= ~PCI_COMMAND_MEMORY;
12789                 pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
12790         }
12791         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
12792                 tp->read32_mbox = tg3_read32_mbox_5906;
12793                 tp->write32_mbox = tg3_write32_mbox_5906;
12794                 tp->write32_tx_mbox = tg3_write32_mbox_5906;
12795                 tp->write32_rx_mbox = tg3_write32_mbox_5906;
12796         }
12797
12798         if (tp->write32 == tg3_write_indirect_reg32 ||
12799             ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
12800              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12801               GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)))
12802                 tp->tg3_flags |= TG3_FLAG_SRAM_USE_CONFIG;
12803
12804         /* Get eeprom hw config before calling tg3_set_power_state().
12805          * In particular, the TG3_FLG2_IS_NIC flag must be
12806          * determined before calling tg3_set_power_state() so that
12807          * we know whether or not to switch out of Vaux power.
12808          * When the flag is set, it means that GPIO1 is used for eeprom
12809          * write protect and also implies that it is a LOM where GPIOs
12810          * are not used to switch power.
12811          */
12812         tg3_get_eeprom_hw_cfg(tp);
12813
12814         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
12815                 /* Allow reads and writes to the
12816                  * APE register and memory space.
12817                  */
12818                 pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
12819                                  PCISTATE_ALLOW_APE_SHMEM_WR;
12820                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
12821                                        pci_state_reg);
12822         }
12823
12824         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12825             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
12826             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
12827             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
12828             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
12829                 tp->tg3_flags |= TG3_FLAG_CPMU_PRESENT;
12830
12831         /* Set up tp->grc_local_ctrl before calling tg3_set_power_state().
12832          * GPIO1 driven high will bring 5700's external PHY out of reset.
12833          * It is also used as eeprom write protect on LOMs.
12834          */
12835         tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
12836         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
12837             (tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT))
12838                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
12839                                        GRC_LCLCTRL_GPIO_OUTPUT1);
12840         /* Unused GPIO3 must be driven as output on 5752 because there
12841          * are no pull-up resistors on unused GPIO pins.
12842          */
12843         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
12844                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
12845
12846         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
12847             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
12848                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
12849
12850         if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
12851             tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
12852                 /* Turn off the debug UART. */
12853                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
12854                 if (tp->tg3_flags2 & TG3_FLG2_IS_NIC)
12855                         /* Keep VMain power. */
12856                         tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
12857                                               GRC_LCLCTRL_GPIO_OUTPUT0;
12858         }
12859
12860         /* Force the chip into D0. */
12861         err = tg3_set_power_state(tp, PCI_D0);
12862         if (err) {
12863                 printk(KERN_ERR PFX "(%s) transition to D0 failed\n",
12864                        pci_name(tp->pdev));
12865                 return err;
12866         }
12867
12868         /* Derive initial jumbo mode from MTU assigned in
12869          * ether_setup() via the alloc_etherdev() call
12870          */
12871         if (tp->dev->mtu > ETH_DATA_LEN &&
12872             !(tp->tg3_flags2 & TG3_FLG2_5780_CLASS))
12873                 tp->tg3_flags |= TG3_FLAG_JUMBO_RING_ENABLE;
12874
12875         /* Determine WakeOnLan speed to use. */
12876         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12877             tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
12878             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0 ||
12879             tp->pci_chip_rev_id == CHIPREV_ID_5701_B2) {
12880                 tp->tg3_flags &= ~(TG3_FLAG_WOL_SPEED_100MB);
12881         } else {
12882                 tp->tg3_flags |= TG3_FLAG_WOL_SPEED_100MB;
12883         }
12884
12885         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
12886                 tp->tg3_flags3 |= TG3_FLG3_PHY_IS_FET;
12887
12888         /* A few boards don't want Ethernet@WireSpeed phy feature */
12889         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
12890             ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
12891              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) &&
12892              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A1)) ||
12893             (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) ||
12894             (tp->tg3_flags2 & TG3_FLG2_ANY_SERDES))
12895                 tp->tg3_flags2 |= TG3_FLG2_NO_ETH_WIRE_SPEED;
12896
12897         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5703_AX ||
12898             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_AX)
12899                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADC_BUG;
12900         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0)
12901                 tp->tg3_flags2 |= TG3_FLG2_PHY_5704_A0_BUG;
12902
12903         if ((tp->tg3_flags2 & TG3_FLG2_5705_PLUS) &&
12904             !(tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) &&
12905             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785 &&
12906             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_57780 &&
12907             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717) {
12908                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
12909                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
12910                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12911                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
12912                         if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
12913                             tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
12914                                 tp->tg3_flags2 |= TG3_FLG2_PHY_JITTER_BUG;
12915                         if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
12916                                 tp->tg3_flags2 |= TG3_FLG2_PHY_ADJUST_TRIM;
12917                 } else
12918                         tp->tg3_flags2 |= TG3_FLG2_PHY_BER_BUG;
12919         }
12920
12921         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
12922             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
12923                 tp->phy_otp = tg3_read_otp_phycfg(tp);
12924                 if (tp->phy_otp == 0)
12925                         tp->phy_otp = TG3_OTP_DEFAULT;
12926         }
12927
12928         if (tp->tg3_flags & TG3_FLAG_CPMU_PRESENT)
12929                 tp->mi_mode = MAC_MI_MODE_500KHZ_CONST;
12930         else
12931                 tp->mi_mode = MAC_MI_MODE_BASE;
12932
12933         tp->coalesce_mode = 0;
12934         if (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_AX &&
12935             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_BX)
12936                 tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
12937
12938         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
12939             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
12940                 tp->tg3_flags3 |= TG3_FLG3_USE_PHYLIB;
12941
12942         if ((tp->pci_chip_rev_id == CHIPREV_ID_57780_A1 &&
12943              tr32(RCVLPC_STATS_ENABLE) & RCVLPC_STATSENAB_ASF_FIX) ||
12944             tp->pci_chip_rev_id == CHIPREV_ID_57780_A0)
12945                 tp->tg3_flags3 |= TG3_FLG3_TOGGLE_10_100_L1PLLPD;
12946
12947         err = tg3_mdio_init(tp);
12948         if (err)
12949                 return err;
12950
12951         /* Initialize data/descriptor byte/word swapping. */
12952         val = tr32(GRC_MODE);
12953         val &= GRC_MODE_HOST_STACKUP;
12954         tw32(GRC_MODE, val | tp->grc_mode);
12955
12956         tg3_switch_clocks(tp);
12957
12958         /* Clear this out for sanity. */
12959         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
12960
12961         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
12962                               &pci_state_reg);
12963         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
12964             (tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) == 0) {
12965                 u32 chiprevid = GET_CHIP_REV_ID(tp->misc_host_ctrl);
12966
12967                 if (chiprevid == CHIPREV_ID_5701_A0 ||
12968                     chiprevid == CHIPREV_ID_5701_B0 ||
12969                     chiprevid == CHIPREV_ID_5701_B2 ||
12970                     chiprevid == CHIPREV_ID_5701_B5) {
12971                         void __iomem *sram_base;
12972
12973                         /* Write some dummy words into the SRAM status block
12974                          * area, see if it reads back correctly.  If the return
12975                          * value is bad, force enable the PCIX workaround.
12976                          */
12977                         sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
12978
12979                         writel(0x00000000, sram_base);
12980                         writel(0x00000000, sram_base + 4);
12981                         writel(0xffffffff, sram_base + 4);
12982                         if (readl(sram_base) != 0x00000000)
12983                                 tp->tg3_flags |= TG3_FLAG_PCIX_TARGET_HWBUG;
12984                 }
12985         }
12986
12987         udelay(50);
12988         tg3_nvram_init(tp);
12989
12990         grc_misc_cfg = tr32(GRC_MISC_CFG);
12991         grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
12992
12993         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
12994             (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
12995              grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
12996                 tp->tg3_flags2 |= TG3_FLG2_IS_5788;
12997
12998         if (!(tp->tg3_flags2 & TG3_FLG2_IS_5788) &&
12999             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700))
13000                 tp->tg3_flags |= TG3_FLAG_TAGGED_STATUS;
13001         if (tp->tg3_flags & TG3_FLAG_TAGGED_STATUS) {
13002                 tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
13003                                       HOSTCC_MODE_CLRTICK_TXBD);
13004
13005                 tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
13006                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
13007                                        tp->misc_host_ctrl);
13008         }
13009
13010         /* Preserve the APE MAC_MODE bits */
13011         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE)
13012                 tp->mac_mode = tr32(MAC_MODE) |
13013                                MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
13014         else
13015                 tp->mac_mode = TG3_DEF_MAC_MODE;
13016
13017         /* these are limited to 10/100 only */
13018         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
13019              (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
13020             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
13021              tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
13022              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901 ||
13023               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901_2 ||
13024               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5705F)) ||
13025             (tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
13026              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5751F ||
13027               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5753F ||
13028               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5787F)) ||
13029             tp->pdev->device == TG3PCI_DEVICE_TIGON3_57790 ||
13030             (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET))
13031                 tp->tg3_flags |= TG3_FLAG_10_100_ONLY;
13032
13033         err = tg3_phy_probe(tp);
13034         if (err) {
13035                 printk(KERN_ERR PFX "(%s) phy probe failed, err %d\n",
13036                        pci_name(tp->pdev), err);
13037                 /* ... but do not return immediately ... */
13038                 tg3_mdio_fini(tp);
13039         }
13040
13041         tg3_read_partno(tp);
13042         tg3_read_fw_ver(tp);
13043
13044         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) {
13045                 tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
13046         } else {
13047                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
13048                         tp->tg3_flags |= TG3_FLAG_USE_MI_INTERRUPT;
13049                 else
13050                         tp->tg3_flags &= ~TG3_FLAG_USE_MI_INTERRUPT;
13051         }
13052
13053         /* 5700 {AX,BX} chips have a broken status block link
13054          * change bit implementation, so we must use the
13055          * status register in those cases.
13056          */
13057         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
13058                 tp->tg3_flags |= TG3_FLAG_USE_LINKCHG_REG;
13059         else
13060                 tp->tg3_flags &= ~TG3_FLAG_USE_LINKCHG_REG;
13061
13062         /* The led_ctrl is set during tg3_phy_probe, here we might
13063          * have to force the link status polling mechanism based
13064          * upon subsystem IDs.
13065          */
13066         if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
13067             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
13068             !(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)) {
13069                 tp->tg3_flags |= (TG3_FLAG_USE_MI_INTERRUPT |
13070                                   TG3_FLAG_USE_LINKCHG_REG);
13071         }
13072
13073         /* For all SERDES we poll the MAC status register. */
13074         if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES)
13075                 tp->tg3_flags |= TG3_FLAG_POLL_SERDES;
13076         else
13077                 tp->tg3_flags &= ~TG3_FLAG_POLL_SERDES;
13078
13079         tp->rx_offset = NET_IP_ALIGN;
13080         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
13081             (tp->tg3_flags & TG3_FLAG_PCIX_MODE) != 0)
13082                 tp->rx_offset = 0;
13083
13084         tp->rx_std_max_post = TG3_RX_RING_SIZE;
13085
13086         /* Increment the rx prod index on the rx std ring by at most
13087          * 8 for these chips to workaround hw errata.
13088          */
13089         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
13090             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
13091             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
13092                 tp->rx_std_max_post = 8;
13093
13094         if (tp->tg3_flags & TG3_FLAG_ASPM_WORKAROUND)
13095                 tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
13096                                      PCIE_PWR_MGMT_L1_THRESH_MSK;
13097
13098         return err;
13099 }
13100
13101 #ifdef CONFIG_SPARC
13102 static int __devinit tg3_get_macaddr_sparc(struct tg3 *tp)
13103 {
13104         struct net_device *dev = tp->dev;
13105         struct pci_dev *pdev = tp->pdev;
13106         struct device_node *dp = pci_device_to_OF_node(pdev);
13107         const unsigned char *addr;
13108         int len;
13109
13110         addr = of_get_property(dp, "local-mac-address", &len);
13111         if (addr && len == 6) {
13112                 memcpy(dev->dev_addr, addr, 6);
13113                 memcpy(dev->perm_addr, dev->dev_addr, 6);
13114                 return 0;
13115         }
13116         return -ENODEV;
13117 }
13118
13119 static int __devinit tg3_get_default_macaddr_sparc(struct tg3 *tp)
13120 {
13121         struct net_device *dev = tp->dev;
13122
13123         memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
13124         memcpy(dev->perm_addr, idprom->id_ethaddr, 6);
13125         return 0;
13126 }
13127 #endif
13128
13129 static int __devinit tg3_get_device_address(struct tg3 *tp)
13130 {
13131         struct net_device *dev = tp->dev;
13132         u32 hi, lo, mac_offset;
13133         int addr_ok = 0;
13134
13135 #ifdef CONFIG_SPARC
13136         if (!tg3_get_macaddr_sparc(tp))
13137                 return 0;
13138 #endif
13139
13140         mac_offset = 0x7c;
13141         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
13142             (tp->tg3_flags2 & TG3_FLG2_5780_CLASS)) {
13143                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
13144                         mac_offset = 0xcc;
13145                 if (tg3_nvram_lock(tp))
13146                         tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
13147                 else
13148                         tg3_nvram_unlock(tp);
13149         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
13150                 if (tr32(TG3_CPMU_STATUS) & TG3_CPMU_STATUS_PCIE_FUNC)
13151                         mac_offset = 0xcc;
13152         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
13153                 mac_offset = 0x10;
13154
13155         /* First try to get it from MAC address mailbox. */
13156         tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
13157         if ((hi >> 16) == 0x484b) {
13158                 dev->dev_addr[0] = (hi >>  8) & 0xff;
13159                 dev->dev_addr[1] = (hi >>  0) & 0xff;
13160
13161                 tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
13162                 dev->dev_addr[2] = (lo >> 24) & 0xff;
13163                 dev->dev_addr[3] = (lo >> 16) & 0xff;
13164                 dev->dev_addr[4] = (lo >>  8) & 0xff;
13165                 dev->dev_addr[5] = (lo >>  0) & 0xff;
13166
13167                 /* Some old bootcode may report a 0 MAC address in SRAM */
13168                 addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
13169         }
13170         if (!addr_ok) {
13171                 /* Next, try NVRAM. */
13172                 if (!(tp->tg3_flags3 & TG3_FLG3_NO_NVRAM) &&
13173                     !tg3_nvram_read_be32(tp, mac_offset + 0, &hi) &&
13174                     !tg3_nvram_read_be32(tp, mac_offset + 4, &lo)) {
13175                         memcpy(&dev->dev_addr[0], ((char *)&hi) + 2, 2);
13176                         memcpy(&dev->dev_addr[2], (char *)&lo, sizeof(lo));
13177                 }
13178                 /* Finally just fetch it out of the MAC control regs. */
13179                 else {
13180                         hi = tr32(MAC_ADDR_0_HIGH);
13181                         lo = tr32(MAC_ADDR_0_LOW);
13182
13183                         dev->dev_addr[5] = lo & 0xff;
13184                         dev->dev_addr[4] = (lo >> 8) & 0xff;
13185                         dev->dev_addr[3] = (lo >> 16) & 0xff;
13186                         dev->dev_addr[2] = (lo >> 24) & 0xff;
13187                         dev->dev_addr[1] = hi & 0xff;
13188                         dev->dev_addr[0] = (hi >> 8) & 0xff;
13189                 }
13190         }
13191
13192         if (!is_valid_ether_addr(&dev->dev_addr[0])) {
13193 #ifdef CONFIG_SPARC
13194                 if (!tg3_get_default_macaddr_sparc(tp))
13195                         return 0;
13196 #endif
13197                 return -EINVAL;
13198         }
13199         memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
13200         return 0;
13201 }
13202
13203 #define BOUNDARY_SINGLE_CACHELINE       1
13204 #define BOUNDARY_MULTI_CACHELINE        2
13205
13206 static u32 __devinit tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
13207 {
13208         int cacheline_size;
13209         u8 byte;
13210         int goal;
13211
13212         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
13213         if (byte == 0)
13214                 cacheline_size = 1024;
13215         else
13216                 cacheline_size = (int) byte * 4;
13217
13218         /* On 5703 and later chips, the boundary bits have no
13219          * effect.
13220          */
13221         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
13222             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
13223             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS))
13224                 goto out;
13225
13226 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
13227         goal = BOUNDARY_MULTI_CACHELINE;
13228 #else
13229 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
13230         goal = BOUNDARY_SINGLE_CACHELINE;
13231 #else
13232         goal = 0;
13233 #endif
13234 #endif
13235
13236         if (!goal)
13237                 goto out;
13238
13239         /* PCI controllers on most RISC systems tend to disconnect
13240          * when a device tries to burst across a cache-line boundary.
13241          * Therefore, letting tg3 do so just wastes PCI bandwidth.
13242          *
13243          * Unfortunately, for PCI-E there are only limited
13244          * write-side controls for this, and thus for reads
13245          * we will still get the disconnects.  We'll also waste
13246          * these PCI cycles for both read and write for chips
13247          * other than 5700 and 5701 which do not implement the
13248          * boundary bits.
13249          */
13250         if ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) &&
13251             !(tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS)) {
13252                 switch (cacheline_size) {
13253                 case 16:
13254                 case 32:
13255                 case 64:
13256                 case 128:
13257                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
13258                                 val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
13259                                         DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
13260                         } else {
13261                                 val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
13262                                         DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
13263                         }
13264                         break;
13265
13266                 case 256:
13267                         val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
13268                                 DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
13269                         break;
13270
13271                 default:
13272                         val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
13273                                 DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
13274                         break;
13275                 }
13276         } else if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
13277                 switch (cacheline_size) {
13278                 case 16:
13279                 case 32:
13280                 case 64:
13281                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
13282                                 val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
13283                                 val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
13284                                 break;
13285                         }
13286                         /* fallthrough */
13287                 case 128:
13288                 default:
13289                         val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
13290                         val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
13291                         break;
13292                 }
13293         } else {
13294                 switch (cacheline_size) {
13295                 case 16:
13296                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
13297                                 val |= (DMA_RWCTRL_READ_BNDRY_16 |
13298                                         DMA_RWCTRL_WRITE_BNDRY_16);
13299                                 break;
13300                         }
13301                         /* fallthrough */
13302                 case 32:
13303                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
13304                                 val |= (DMA_RWCTRL_READ_BNDRY_32 |
13305                                         DMA_RWCTRL_WRITE_BNDRY_32);
13306                                 break;
13307                         }
13308                         /* fallthrough */
13309                 case 64:
13310                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
13311                                 val |= (DMA_RWCTRL_READ_BNDRY_64 |
13312                                         DMA_RWCTRL_WRITE_BNDRY_64);
13313                                 break;
13314                         }
13315                         /* fallthrough */
13316                 case 128:
13317                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
13318                                 val |= (DMA_RWCTRL_READ_BNDRY_128 |
13319                                         DMA_RWCTRL_WRITE_BNDRY_128);
13320                                 break;
13321                         }
13322                         /* fallthrough */
13323                 case 256:
13324                         val |= (DMA_RWCTRL_READ_BNDRY_256 |
13325                                 DMA_RWCTRL_WRITE_BNDRY_256);
13326                         break;
13327                 case 512:
13328                         val |= (DMA_RWCTRL_READ_BNDRY_512 |
13329                                 DMA_RWCTRL_WRITE_BNDRY_512);
13330                         break;
13331                 case 1024:
13332                 default:
13333                         val |= (DMA_RWCTRL_READ_BNDRY_1024 |
13334                                 DMA_RWCTRL_WRITE_BNDRY_1024);
13335                         break;
13336                 }
13337         }
13338
13339 out:
13340         return val;
13341 }
13342
13343 static int __devinit tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma, int size, int to_device)
13344 {
13345         struct tg3_internal_buffer_desc test_desc;
13346         u32 sram_dma_descs;
13347         int i, ret;
13348
13349         sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
13350
13351         tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
13352         tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
13353         tw32(RDMAC_STATUS, 0);
13354         tw32(WDMAC_STATUS, 0);
13355
13356         tw32(BUFMGR_MODE, 0);
13357         tw32(FTQ_RESET, 0);
13358
13359         test_desc.addr_hi = ((u64) buf_dma) >> 32;
13360         test_desc.addr_lo = buf_dma & 0xffffffff;
13361         test_desc.nic_mbuf = 0x00002100;
13362         test_desc.len = size;
13363
13364         /*
13365          * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
13366          * the *second* time the tg3 driver was getting loaded after an
13367          * initial scan.
13368          *
13369          * Broadcom tells me:
13370          *   ...the DMA engine is connected to the GRC block and a DMA
13371          *   reset may affect the GRC block in some unpredictable way...
13372          *   The behavior of resets to individual blocks has not been tested.
13373          *
13374          * Broadcom noted the GRC reset will also reset all sub-components.
13375          */
13376         if (to_device) {
13377                 test_desc.cqid_sqid = (13 << 8) | 2;
13378
13379                 tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
13380                 udelay(40);
13381         } else {
13382                 test_desc.cqid_sqid = (16 << 8) | 7;
13383
13384                 tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
13385                 udelay(40);
13386         }
13387         test_desc.flags = 0x00000005;
13388
13389         for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
13390                 u32 val;
13391
13392                 val = *(((u32 *)&test_desc) + i);
13393                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
13394                                        sram_dma_descs + (i * sizeof(u32)));
13395                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
13396         }
13397         pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
13398
13399         if (to_device) {
13400                 tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
13401         } else {
13402                 tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
13403         }
13404
13405         ret = -ENODEV;
13406         for (i = 0; i < 40; i++) {
13407                 u32 val;
13408
13409                 if (to_device)
13410                         val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
13411                 else
13412                         val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
13413                 if ((val & 0xffff) == sram_dma_descs) {
13414                         ret = 0;
13415                         break;
13416                 }
13417
13418                 udelay(100);
13419         }
13420
13421         return ret;
13422 }
13423
13424 #define TEST_BUFFER_SIZE        0x2000
13425
13426 static int __devinit tg3_test_dma(struct tg3 *tp)
13427 {
13428         dma_addr_t buf_dma;
13429         u32 *buf, saved_dma_rwctrl;
13430         int ret;
13431
13432         buf = pci_alloc_consistent(tp->pdev, TEST_BUFFER_SIZE, &buf_dma);
13433         if (!buf) {
13434                 ret = -ENOMEM;
13435                 goto out_nofree;
13436         }
13437
13438         tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
13439                           (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
13440
13441         tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
13442
13443         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
13444                 /* DMA read watermark not used on PCIE */
13445                 tp->dma_rwctrl |= 0x00180000;
13446         } else if (!(tp->tg3_flags & TG3_FLAG_PCIX_MODE)) {
13447                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
13448                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)
13449                         tp->dma_rwctrl |= 0x003f0000;
13450                 else
13451                         tp->dma_rwctrl |= 0x003f000f;
13452         } else {
13453                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
13454                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
13455                         u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
13456                         u32 read_water = 0x7;
13457
13458                         /* If the 5704 is behind the EPB bridge, we can
13459                          * do the less restrictive ONE_DMA workaround for
13460                          * better performance.
13461                          */
13462                         if ((tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) &&
13463                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
13464                                 tp->dma_rwctrl |= 0x8000;
13465                         else if (ccval == 0x6 || ccval == 0x7)
13466                                 tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
13467
13468                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703)
13469                                 read_water = 4;
13470                         /* Set bit 23 to enable PCIX hw bug fix */
13471                         tp->dma_rwctrl |=
13472                                 (read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
13473                                 (0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
13474                                 (1 << 23);
13475                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780) {
13476                         /* 5780 always in PCIX mode */
13477                         tp->dma_rwctrl |= 0x00144000;
13478                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
13479                         /* 5714 always in PCIX mode */
13480                         tp->dma_rwctrl |= 0x00148000;
13481                 } else {
13482                         tp->dma_rwctrl |= 0x001b000f;
13483                 }
13484         }
13485
13486         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
13487             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
13488                 tp->dma_rwctrl &= 0xfffffff0;
13489
13490         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
13491             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
13492                 /* Remove this if it causes problems for some boards. */
13493                 tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
13494
13495                 /* On 5700/5701 chips, we need to set this bit.
13496                  * Otherwise the chip will issue cacheline transactions
13497                  * to streamable DMA memory with not all the byte
13498                  * enables turned on.  This is an error on several
13499                  * RISC PCI controllers, in particular sparc64.
13500                  *
13501                  * On 5703/5704 chips, this bit has been reassigned
13502                  * a different meaning.  In particular, it is used
13503                  * on those chips to enable a PCI-X workaround.
13504                  */
13505                 tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
13506         }
13507
13508         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
13509
13510 #if 0
13511         /* Unneeded, already done by tg3_get_invariants.  */
13512         tg3_switch_clocks(tp);
13513 #endif
13514
13515         ret = 0;
13516         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
13517             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
13518                 goto out;
13519
13520         /* It is best to perform DMA test with maximum write burst size
13521          * to expose the 5700/5701 write DMA bug.
13522          */
13523         saved_dma_rwctrl = tp->dma_rwctrl;
13524         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
13525         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
13526
13527         while (1) {
13528                 u32 *p = buf, i;
13529
13530                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
13531                         p[i] = i;
13532
13533                 /* Send the buffer to the chip. */
13534                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 1);
13535                 if (ret) {
13536                         printk(KERN_ERR "tg3_test_dma() Write the buffer failed %d\n", ret);
13537                         break;
13538                 }
13539
13540 #if 0
13541                 /* validate data reached card RAM correctly. */
13542                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
13543                         u32 val;
13544                         tg3_read_mem(tp, 0x2100 + (i*4), &val);
13545                         if (le32_to_cpu(val) != p[i]) {
13546                                 printk(KERN_ERR "  tg3_test_dma()  Card buffer corrupted on write! (%d != %d)\n", val, i);
13547                                 /* ret = -ENODEV here? */
13548                         }
13549                         p[i] = 0;
13550                 }
13551 #endif
13552                 /* Now read it back. */
13553                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 0);
13554                 if (ret) {
13555                         printk(KERN_ERR "tg3_test_dma() Read the buffer failed %d\n", ret);
13556
13557                         break;
13558                 }
13559
13560                 /* Verify it. */
13561                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
13562                         if (p[i] == i)
13563                                 continue;
13564
13565                         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
13566                             DMA_RWCTRL_WRITE_BNDRY_16) {
13567                                 tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
13568                                 tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
13569                                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
13570                                 break;
13571                         } else {
13572                                 printk(KERN_ERR "tg3_test_dma() buffer corrupted on read back! (%d != %d)\n", p[i], i);
13573                                 ret = -ENODEV;
13574                                 goto out;
13575                         }
13576                 }
13577
13578                 if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
13579                         /* Success. */
13580                         ret = 0;
13581                         break;
13582                 }
13583         }
13584         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
13585             DMA_RWCTRL_WRITE_BNDRY_16) {
13586                 static struct pci_device_id dma_wait_state_chipsets[] = {
13587                         { PCI_DEVICE(PCI_VENDOR_ID_APPLE,
13588                                      PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
13589                         { },
13590                 };
13591
13592                 /* DMA test passed without adjusting DMA boundary,
13593                  * now look for chipsets that are known to expose the
13594                  * DMA bug without failing the test.
13595                  */
13596                 if (pci_dev_present(dma_wait_state_chipsets)) {
13597                         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
13598                         tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
13599                 }
13600                 else
13601                         /* Safe to use the calculated DMA boundary. */
13602                         tp->dma_rwctrl = saved_dma_rwctrl;
13603
13604                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
13605         }
13606
13607 out:
13608         pci_free_consistent(tp->pdev, TEST_BUFFER_SIZE, buf, buf_dma);
13609 out_nofree:
13610         return ret;
13611 }
13612
13613 static void __devinit tg3_init_link_config(struct tg3 *tp)
13614 {
13615         tp->link_config.advertising =
13616                 (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
13617                  ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
13618                  ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
13619                  ADVERTISED_Autoneg | ADVERTISED_MII);
13620         tp->link_config.speed = SPEED_INVALID;
13621         tp->link_config.duplex = DUPLEX_INVALID;
13622         tp->link_config.autoneg = AUTONEG_ENABLE;
13623         tp->link_config.active_speed = SPEED_INVALID;
13624         tp->link_config.active_duplex = DUPLEX_INVALID;
13625         tp->link_config.phy_is_low_power = 0;
13626         tp->link_config.orig_speed = SPEED_INVALID;
13627         tp->link_config.orig_duplex = DUPLEX_INVALID;
13628         tp->link_config.orig_autoneg = AUTONEG_INVALID;
13629 }
13630
13631 static void __devinit tg3_init_bufmgr_config(struct tg3 *tp)
13632 {
13633         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS &&
13634             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717) {
13635                 tp->bufmgr_config.mbuf_read_dma_low_water =
13636                         DEFAULT_MB_RDMA_LOW_WATER_5705;
13637                 tp->bufmgr_config.mbuf_mac_rx_low_water =
13638                         DEFAULT_MB_MACRX_LOW_WATER_5705;
13639                 tp->bufmgr_config.mbuf_high_water =
13640                         DEFAULT_MB_HIGH_WATER_5705;
13641                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
13642                         tp->bufmgr_config.mbuf_mac_rx_low_water =
13643                                 DEFAULT_MB_MACRX_LOW_WATER_5906;
13644                         tp->bufmgr_config.mbuf_high_water =
13645                                 DEFAULT_MB_HIGH_WATER_5906;
13646                 }
13647
13648                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
13649                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
13650                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
13651                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
13652                 tp->bufmgr_config.mbuf_high_water_jumbo =
13653                         DEFAULT_MB_HIGH_WATER_JUMBO_5780;
13654         } else {
13655                 tp->bufmgr_config.mbuf_read_dma_low_water =
13656                         DEFAULT_MB_RDMA_LOW_WATER;
13657                 tp->bufmgr_config.mbuf_mac_rx_low_water =
13658                         DEFAULT_MB_MACRX_LOW_WATER;
13659                 tp->bufmgr_config.mbuf_high_water =
13660                         DEFAULT_MB_HIGH_WATER;
13661
13662                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
13663                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
13664                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
13665                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
13666                 tp->bufmgr_config.mbuf_high_water_jumbo =
13667                         DEFAULT_MB_HIGH_WATER_JUMBO;
13668         }
13669
13670         tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
13671         tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
13672 }
13673
13674 static char * __devinit tg3_phy_string(struct tg3 *tp)
13675 {
13676         switch (tp->phy_id & PHY_ID_MASK) {
13677         case PHY_ID_BCM5400:    return "5400";
13678         case PHY_ID_BCM5401:    return "5401";
13679         case PHY_ID_BCM5411:    return "5411";
13680         case PHY_ID_BCM5701:    return "5701";
13681         case PHY_ID_BCM5703:    return "5703";
13682         case PHY_ID_BCM5704:    return "5704";
13683         case PHY_ID_BCM5705:    return "5705";
13684         case PHY_ID_BCM5750:    return "5750";
13685         case PHY_ID_BCM5752:    return "5752";
13686         case PHY_ID_BCM5714:    return "5714";
13687         case PHY_ID_BCM5780:    return "5780";
13688         case PHY_ID_BCM5755:    return "5755";
13689         case PHY_ID_BCM5787:    return "5787";
13690         case PHY_ID_BCM5784:    return "5784";
13691         case PHY_ID_BCM5756:    return "5722/5756";
13692         case PHY_ID_BCM5906:    return "5906";
13693         case PHY_ID_BCM5761:    return "5761";
13694         case PHY_ID_BCM8002:    return "8002/serdes";
13695         case 0:                 return "serdes";
13696         default:                return "unknown";
13697         }
13698 }
13699
13700 static char * __devinit tg3_bus_string(struct tg3 *tp, char *str)
13701 {
13702         if (tp->tg3_flags2 & TG3_FLG2_PCI_EXPRESS) {
13703                 strcpy(str, "PCI Express");
13704                 return str;
13705         } else if (tp->tg3_flags & TG3_FLAG_PCIX_MODE) {
13706                 u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
13707
13708                 strcpy(str, "PCIX:");
13709
13710                 if ((clock_ctrl == 7) ||
13711                     ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
13712                      GRC_MISC_CFG_BOARD_ID_5704CIOBE))
13713                         strcat(str, "133MHz");
13714                 else if (clock_ctrl == 0)
13715                         strcat(str, "33MHz");
13716                 else if (clock_ctrl == 2)
13717                         strcat(str, "50MHz");
13718                 else if (clock_ctrl == 4)
13719                         strcat(str, "66MHz");
13720                 else if (clock_ctrl == 6)
13721                         strcat(str, "100MHz");
13722         } else {
13723                 strcpy(str, "PCI:");
13724                 if (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED)
13725                         strcat(str, "66MHz");
13726                 else
13727                         strcat(str, "33MHz");
13728         }
13729         if (tp->tg3_flags & TG3_FLAG_PCI_32BIT)
13730                 strcat(str, ":32-bit");
13731         else
13732                 strcat(str, ":64-bit");
13733         return str;
13734 }
13735
13736 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *tp)
13737 {
13738         struct pci_dev *peer;
13739         unsigned int func, devnr = tp->pdev->devfn & ~7;
13740
13741         for (func = 0; func < 8; func++) {
13742                 peer = pci_get_slot(tp->pdev->bus, devnr | func);
13743                 if (peer && peer != tp->pdev)
13744                         break;
13745                 pci_dev_put(peer);
13746         }
13747         /* 5704 can be configured in single-port mode, set peer to
13748          * tp->pdev in that case.
13749          */
13750         if (!peer) {
13751                 peer = tp->pdev;
13752                 return peer;
13753         }
13754
13755         /*
13756          * We don't need to keep the refcount elevated; there's no way
13757          * to remove one half of this device without removing the other
13758          */
13759         pci_dev_put(peer);
13760
13761         return peer;
13762 }
13763
13764 static void __devinit tg3_init_coal(struct tg3 *tp)
13765 {
13766         struct ethtool_coalesce *ec = &tp->coal;
13767
13768         memset(ec, 0, sizeof(*ec));
13769         ec->cmd = ETHTOOL_GCOALESCE;
13770         ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
13771         ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
13772         ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
13773         ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
13774         ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
13775         ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
13776         ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
13777         ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
13778         ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
13779
13780         if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
13781                                  HOSTCC_MODE_CLRTICK_TXBD)) {
13782                 ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
13783                 ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
13784                 ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
13785                 ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
13786         }
13787
13788         if (tp->tg3_flags2 & TG3_FLG2_5705_PLUS) {
13789                 ec->rx_coalesce_usecs_irq = 0;
13790                 ec->tx_coalesce_usecs_irq = 0;
13791                 ec->stats_block_coalesce_usecs = 0;
13792         }
13793 }
13794
13795 static const struct net_device_ops tg3_netdev_ops = {
13796         .ndo_open               = tg3_open,
13797         .ndo_stop               = tg3_close,
13798         .ndo_start_xmit         = tg3_start_xmit,
13799         .ndo_get_stats          = tg3_get_stats,
13800         .ndo_validate_addr      = eth_validate_addr,
13801         .ndo_set_multicast_list = tg3_set_rx_mode,
13802         .ndo_set_mac_address    = tg3_set_mac_addr,
13803         .ndo_do_ioctl           = tg3_ioctl,
13804         .ndo_tx_timeout         = tg3_tx_timeout,
13805         .ndo_change_mtu         = tg3_change_mtu,
13806 #if TG3_VLAN_TAG_USED
13807         .ndo_vlan_rx_register   = tg3_vlan_rx_register,
13808 #endif
13809 #ifdef CONFIG_NET_POLL_CONTROLLER
13810         .ndo_poll_controller    = tg3_poll_controller,
13811 #endif
13812 };
13813
13814 static const struct net_device_ops tg3_netdev_ops_dma_bug = {
13815         .ndo_open               = tg3_open,
13816         .ndo_stop               = tg3_close,
13817         .ndo_start_xmit         = tg3_start_xmit_dma_bug,
13818         .ndo_get_stats          = tg3_get_stats,
13819         .ndo_validate_addr      = eth_validate_addr,
13820         .ndo_set_multicast_list = tg3_set_rx_mode,
13821         .ndo_set_mac_address    = tg3_set_mac_addr,
13822         .ndo_do_ioctl           = tg3_ioctl,
13823         .ndo_tx_timeout         = tg3_tx_timeout,
13824         .ndo_change_mtu         = tg3_change_mtu,
13825 #if TG3_VLAN_TAG_USED
13826         .ndo_vlan_rx_register   = tg3_vlan_rx_register,
13827 #endif
13828 #ifdef CONFIG_NET_POLL_CONTROLLER
13829         .ndo_poll_controller    = tg3_poll_controller,
13830 #endif
13831 };
13832
13833 static int __devinit tg3_init_one(struct pci_dev *pdev,
13834                                   const struct pci_device_id *ent)
13835 {
13836         static int tg3_version_printed = 0;
13837         struct net_device *dev;
13838         struct tg3 *tp;
13839         int i, err, pm_cap;
13840         u32 sndmbx, rcvmbx, intmbx;
13841         char str[40];
13842         u64 dma_mask, persist_dma_mask;
13843
13844         if (tg3_version_printed++ == 0)
13845                 printk(KERN_INFO "%s", version);
13846
13847         err = pci_enable_device(pdev);
13848         if (err) {
13849                 printk(KERN_ERR PFX "Cannot enable PCI device, "
13850                        "aborting.\n");
13851                 return err;
13852         }
13853
13854         err = pci_request_regions(pdev, DRV_MODULE_NAME);
13855         if (err) {
13856                 printk(KERN_ERR PFX "Cannot obtain PCI resources, "
13857                        "aborting.\n");
13858                 goto err_out_disable_pdev;
13859         }
13860
13861         pci_set_master(pdev);
13862
13863         /* Find power-management capability. */
13864         pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
13865         if (pm_cap == 0) {
13866                 printk(KERN_ERR PFX "Cannot find PowerManagement capability, "
13867                        "aborting.\n");
13868                 err = -EIO;
13869                 goto err_out_free_res;
13870         }
13871
13872         dev = alloc_etherdev_mq(sizeof(*tp), TG3_IRQ_MAX_VECS);
13873         if (!dev) {
13874                 printk(KERN_ERR PFX "Etherdev alloc failed, aborting.\n");
13875                 err = -ENOMEM;
13876                 goto err_out_free_res;
13877         }
13878
13879         SET_NETDEV_DEV(dev, &pdev->dev);
13880
13881 #if TG3_VLAN_TAG_USED
13882         dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
13883 #endif
13884
13885         tp = netdev_priv(dev);
13886         tp->pdev = pdev;
13887         tp->dev = dev;
13888         tp->pm_cap = pm_cap;
13889         tp->rx_mode = TG3_DEF_RX_MODE;
13890         tp->tx_mode = TG3_DEF_TX_MODE;
13891
13892         if (tg3_debug > 0)
13893                 tp->msg_enable = tg3_debug;
13894         else
13895                 tp->msg_enable = TG3_DEF_MSG_ENABLE;
13896
13897         /* The word/byte swap controls here control register access byte
13898          * swapping.  DMA data byte swapping is controlled in the GRC_MODE
13899          * setting below.
13900          */
13901         tp->misc_host_ctrl =
13902                 MISC_HOST_CTRL_MASK_PCI_INT |
13903                 MISC_HOST_CTRL_WORD_SWAP |
13904                 MISC_HOST_CTRL_INDIR_ACCESS |
13905                 MISC_HOST_CTRL_PCISTATE_RW;
13906
13907         /* The NONFRM (non-frame) byte/word swap controls take effect
13908          * on descriptor entries, anything which isn't packet data.
13909          *
13910          * The StrongARM chips on the board (one for tx, one for rx)
13911          * are running in big-endian mode.
13912          */
13913         tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
13914                         GRC_MODE_WSWAP_NONFRM_DATA);
13915 #ifdef __BIG_ENDIAN
13916         tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
13917 #endif
13918         spin_lock_init(&tp->lock);
13919         spin_lock_init(&tp->indirect_lock);
13920         INIT_WORK(&tp->reset_task, tg3_reset_task);
13921
13922         tp->regs = pci_ioremap_bar(pdev, BAR_0);
13923         if (!tp->regs) {
13924                 printk(KERN_ERR PFX "Cannot map device registers, "
13925                        "aborting.\n");
13926                 err = -ENOMEM;
13927                 goto err_out_free_dev;
13928         }
13929
13930         tg3_init_link_config(tp);
13931
13932         tp->rx_pending = TG3_DEF_RX_RING_PENDING;
13933         tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
13934
13935         intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW;
13936         rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW;
13937         sndmbx = MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW;
13938         for (i = 0; i < TG3_IRQ_MAX_VECS; i++) {
13939                 struct tg3_napi *tnapi = &tp->napi[i];
13940
13941                 tnapi->tp = tp;
13942                 tnapi->tx_pending = TG3_DEF_TX_RING_PENDING;
13943
13944                 tnapi->int_mbox = intmbx;
13945                 if (i < 4)
13946                         intmbx += 0x8;
13947                 else
13948                         intmbx += 0x4;
13949
13950                 tnapi->consmbox = rcvmbx;
13951                 tnapi->prodmbox = sndmbx;
13952
13953                 if (i)
13954                         tnapi->coal_now = HOSTCC_MODE_COAL_VEC1_NOW << (i - 1);
13955                 else
13956                         tnapi->coal_now = HOSTCC_MODE_NOW;
13957
13958                 if (!(tp->tg3_flags & TG3_FLAG_SUPPORT_MSIX))
13959                         break;
13960
13961                 /*
13962                  * If we support MSIX, we'll be using RSS.  If we're using
13963                  * RSS, the first vector only handles link interrupts and the
13964                  * remaining vectors handle rx and tx interrupts.  Reuse the
13965                  * mailbox values for the next iteration.  The values we setup
13966                  * above are still useful for the single vectored mode.
13967                  */
13968                 if (!i)
13969                         continue;
13970
13971                 rcvmbx += 0x8;
13972
13973                 if (sndmbx & 0x4)
13974                         sndmbx -= 0x4;
13975                 else
13976                         sndmbx += 0xc;
13977         }
13978
13979         netif_napi_add(dev, &tp->napi[0].napi, tg3_poll, 64);
13980         dev->ethtool_ops = &tg3_ethtool_ops;
13981         dev->watchdog_timeo = TG3_TX_TIMEOUT;
13982         dev->irq = pdev->irq;
13983
13984         err = tg3_get_invariants(tp);
13985         if (err) {
13986                 printk(KERN_ERR PFX "Problem fetching invariants of chip, "
13987                        "aborting.\n");
13988                 goto err_out_iounmap;
13989         }
13990
13991         if ((tp->tg3_flags3 & TG3_FLG3_5755_PLUS) ||
13992             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
13993                 dev->netdev_ops = &tg3_netdev_ops;
13994         else
13995                 dev->netdev_ops = &tg3_netdev_ops_dma_bug;
13996
13997
13998         /* The EPB bridge inside 5714, 5715, and 5780 and any
13999          * device behind the EPB cannot support DMA addresses > 40-bit.
14000          * On 64-bit systems with IOMMU, use 40-bit dma_mask.
14001          * On 64-bit systems without IOMMU, use 64-bit dma_mask and
14002          * do DMA address check in tg3_start_xmit().
14003          */
14004         if (tp->tg3_flags2 & TG3_FLG2_IS_5788)
14005                 persist_dma_mask = dma_mask = DMA_BIT_MASK(32);
14006         else if (tp->tg3_flags & TG3_FLAG_40BIT_DMA_BUG) {
14007                 persist_dma_mask = dma_mask = DMA_BIT_MASK(40);
14008 #ifdef CONFIG_HIGHMEM
14009                 dma_mask = DMA_BIT_MASK(64);
14010 #endif
14011         } else
14012                 persist_dma_mask = dma_mask = DMA_BIT_MASK(64);
14013
14014         /* Configure DMA attributes. */
14015         if (dma_mask > DMA_BIT_MASK(32)) {
14016                 err = pci_set_dma_mask(pdev, dma_mask);
14017                 if (!err) {
14018                         dev->features |= NETIF_F_HIGHDMA;
14019                         err = pci_set_consistent_dma_mask(pdev,
14020                                                           persist_dma_mask);
14021                         if (err < 0) {
14022                                 printk(KERN_ERR PFX "Unable to obtain 64 bit "
14023                                        "DMA for consistent allocations\n");
14024                                 goto err_out_iounmap;
14025                         }
14026                 }
14027         }
14028         if (err || dma_mask == DMA_BIT_MASK(32)) {
14029                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
14030                 if (err) {
14031                         printk(KERN_ERR PFX "No usable DMA configuration, "
14032                                "aborting.\n");
14033                         goto err_out_iounmap;
14034                 }
14035         }
14036
14037         tg3_init_bufmgr_config(tp);
14038
14039         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0)
14040                 tp->fw_needed = FIRMWARE_TG3;
14041
14042         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
14043                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE;
14044         }
14045         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
14046             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
14047             tp->pci_chip_rev_id == CHIPREV_ID_5705_A0 ||
14048             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
14049             (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0) {
14050                 tp->tg3_flags2 &= ~TG3_FLG2_TSO_CAPABLE;
14051         } else {
14052                 tp->tg3_flags2 |= TG3_FLG2_TSO_CAPABLE | TG3_FLG2_TSO_BUG;
14053                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)
14054                         tp->fw_needed = FIRMWARE_TG3TSO5;
14055                 else
14056                         tp->fw_needed = FIRMWARE_TG3TSO;
14057         }
14058
14059         /* TSO is on by default on chips that support hardware TSO.
14060          * Firmware TSO on older chips gives lower performance, so it
14061          * is off by default, but can be enabled using ethtool.
14062          */
14063         if (tp->tg3_flags2 & TG3_FLG2_HW_TSO) {
14064                 if (dev->features & NETIF_F_IP_CSUM)
14065                         dev->features |= NETIF_F_TSO;
14066                 if ((dev->features & NETIF_F_IPV6_CSUM) &&
14067                     (tp->tg3_flags2 & TG3_FLG2_HW_TSO_2))
14068                         dev->features |= NETIF_F_TSO6;
14069                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
14070                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
14071                      GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) ||
14072                         GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
14073                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
14074                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
14075                         dev->features |= NETIF_F_TSO_ECN;
14076         }
14077
14078
14079         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 &&
14080             !(tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) &&
14081             !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
14082                 tp->tg3_flags2 |= TG3_FLG2_MAX_RXPEND_64;
14083                 tp->rx_pending = 63;
14084         }
14085
14086         err = tg3_get_device_address(tp);
14087         if (err) {
14088                 printk(KERN_ERR PFX "Could not obtain valid ethernet address, "
14089                        "aborting.\n");
14090                 goto err_out_fw;
14091         }
14092
14093         if (tp->tg3_flags3 & TG3_FLG3_ENABLE_APE) {
14094                 tp->aperegs = pci_ioremap_bar(pdev, BAR_2);
14095                 if (!tp->aperegs) {
14096                         printk(KERN_ERR PFX "Cannot map APE registers, "
14097                                "aborting.\n");
14098                         err = -ENOMEM;
14099                         goto err_out_fw;
14100                 }
14101
14102                 tg3_ape_lock_init(tp);
14103
14104                 if (tp->tg3_flags & TG3_FLAG_ENABLE_ASF)
14105                         tg3_read_dash_ver(tp);
14106         }
14107
14108         /*
14109          * Reset chip in case UNDI or EFI driver did not shutdown
14110          * DMA self test will enable WDMAC and we'll see (spurious)
14111          * pending DMA on the PCI bus at that point.
14112          */
14113         if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
14114             (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
14115                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
14116                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
14117         }
14118
14119         err = tg3_test_dma(tp);
14120         if (err) {
14121                 printk(KERN_ERR PFX "DMA engine test failed, aborting.\n");
14122                 goto err_out_apeunmap;
14123         }
14124
14125         /* flow control autonegotiation is default behavior */
14126         tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG;
14127         tp->link_config.flowctrl = FLOW_CTRL_TX | FLOW_CTRL_RX;
14128
14129         tg3_init_coal(tp);
14130
14131         pci_set_drvdata(pdev, dev);
14132
14133         err = register_netdev(dev);
14134         if (err) {
14135                 printk(KERN_ERR PFX "Cannot register net device, "
14136                        "aborting.\n");
14137                 goto err_out_apeunmap;
14138         }
14139
14140         printk(KERN_INFO "%s: Tigon3 [partno(%s) rev %04x] (%s) MAC address %pM\n",
14141                dev->name,
14142                tp->board_part_number,
14143                tp->pci_chip_rev_id,
14144                tg3_bus_string(tp, str),
14145                dev->dev_addr);
14146
14147         if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED)
14148                 printk(KERN_INFO
14149                        "%s: attached PHY driver [%s] (mii_bus:phy_addr=%s)\n",
14150                        tp->dev->name,
14151                        tp->mdio_bus->phy_map[PHY_ADDR]->drv->name,
14152                        dev_name(&tp->mdio_bus->phy_map[PHY_ADDR]->dev));
14153         else
14154                 printk(KERN_INFO
14155                        "%s: attached PHY is %s (%s Ethernet) (WireSpeed[%d])\n",
14156                        tp->dev->name, tg3_phy_string(tp),
14157                        ((tp->tg3_flags & TG3_FLAG_10_100_ONLY) ? "10/100Base-TX" :
14158                         ((tp->tg3_flags2 & TG3_FLG2_ANY_SERDES) ? "1000Base-SX" :
14159                          "10/100/1000Base-T")),
14160                        (tp->tg3_flags2 & TG3_FLG2_NO_ETH_WIRE_SPEED) == 0);
14161
14162         printk(KERN_INFO "%s: RXcsums[%d] LinkChgREG[%d] MIirq[%d] ASF[%d] TSOcap[%d]\n",
14163                dev->name,
14164                (tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) != 0,
14165                (tp->tg3_flags & TG3_FLAG_USE_LINKCHG_REG) != 0,
14166                (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT) != 0,
14167                (tp->tg3_flags & TG3_FLAG_ENABLE_ASF) != 0,
14168                (tp->tg3_flags2 & TG3_FLG2_TSO_CAPABLE) != 0);
14169         printk(KERN_INFO "%s: dma_rwctrl[%08x] dma_mask[%d-bit]\n",
14170                dev->name, tp->dma_rwctrl,
14171                (pdev->dma_mask == DMA_BIT_MASK(32)) ? 32 :
14172                 (((u64) pdev->dma_mask == DMA_BIT_MASK(40)) ? 40 : 64));
14173
14174         return 0;
14175
14176 err_out_apeunmap:
14177         if (tp->aperegs) {
14178                 iounmap(tp->aperegs);
14179                 tp->aperegs = NULL;
14180         }
14181
14182 err_out_fw:
14183         if (tp->fw)
14184                 release_firmware(tp->fw);
14185
14186 err_out_iounmap:
14187         if (tp->regs) {
14188                 iounmap(tp->regs);
14189                 tp->regs = NULL;
14190         }
14191
14192 err_out_free_dev:
14193         free_netdev(dev);
14194
14195 err_out_free_res:
14196         pci_release_regions(pdev);
14197
14198 err_out_disable_pdev:
14199         pci_disable_device(pdev);
14200         pci_set_drvdata(pdev, NULL);
14201         return err;
14202 }
14203
14204 static void __devexit tg3_remove_one(struct pci_dev *pdev)
14205 {
14206         struct net_device *dev = pci_get_drvdata(pdev);
14207
14208         if (dev) {
14209                 struct tg3 *tp = netdev_priv(dev);
14210
14211                 if (tp->fw)
14212                         release_firmware(tp->fw);
14213
14214                 flush_scheduled_work();
14215
14216                 if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) {
14217                         tg3_phy_fini(tp);
14218                         tg3_mdio_fini(tp);
14219                 }
14220
14221                 unregister_netdev(dev);
14222                 if (tp->aperegs) {
14223                         iounmap(tp->aperegs);
14224                         tp->aperegs = NULL;
14225                 }
14226                 if (tp->regs) {
14227                         iounmap(tp->regs);
14228                         tp->regs = NULL;
14229                 }
14230                 free_netdev(dev);
14231                 pci_release_regions(pdev);
14232                 pci_disable_device(pdev);
14233                 pci_set_drvdata(pdev, NULL);
14234         }
14235 }
14236
14237 static int tg3_suspend(struct pci_dev *pdev, pm_message_t state)
14238 {
14239         struct net_device *dev = pci_get_drvdata(pdev);
14240         struct tg3 *tp = netdev_priv(dev);
14241         pci_power_t target_state;
14242         int err;
14243
14244         /* PCI register 4 needs to be saved whether netif_running() or not.
14245          * MSI address and data need to be saved if using MSI and
14246          * netif_running().
14247          */
14248         pci_save_state(pdev);
14249
14250         if (!netif_running(dev))
14251                 return 0;
14252
14253         flush_scheduled_work();
14254         tg3_phy_stop(tp);
14255         tg3_netif_stop(tp);
14256
14257         del_timer_sync(&tp->timer);
14258
14259         tg3_full_lock(tp, 1);
14260         tg3_disable_ints(tp);
14261         tg3_full_unlock(tp);
14262
14263         netif_device_detach(dev);
14264
14265         tg3_full_lock(tp, 0);
14266         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
14267         tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE;
14268         tg3_full_unlock(tp);
14269
14270         target_state = pdev->pm_cap ? pci_target_state(pdev) : PCI_D3hot;
14271
14272         err = tg3_set_power_state(tp, target_state);
14273         if (err) {
14274                 int err2;
14275
14276                 tg3_full_lock(tp, 0);
14277
14278                 tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
14279                 err2 = tg3_restart_hw(tp, 1);
14280                 if (err2)
14281                         goto out;
14282
14283                 tp->timer.expires = jiffies + tp->timer_offset;
14284                 add_timer(&tp->timer);
14285
14286                 netif_device_attach(dev);
14287                 tg3_netif_start(tp);
14288
14289 out:
14290                 tg3_full_unlock(tp);
14291
14292                 if (!err2)
14293                         tg3_phy_start(tp);
14294         }
14295
14296         return err;
14297 }
14298
14299 static int tg3_resume(struct pci_dev *pdev)
14300 {
14301         struct net_device *dev = pci_get_drvdata(pdev);
14302         struct tg3 *tp = netdev_priv(dev);
14303         int err;
14304
14305         pci_restore_state(tp->pdev);
14306
14307         if (!netif_running(dev))
14308                 return 0;
14309
14310         err = tg3_set_power_state(tp, PCI_D0);
14311         if (err)
14312                 return err;
14313
14314         netif_device_attach(dev);
14315
14316         tg3_full_lock(tp, 0);
14317
14318         tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE;
14319         err = tg3_restart_hw(tp, 1);
14320         if (err)
14321                 goto out;
14322
14323         tp->timer.expires = jiffies + tp->timer_offset;
14324         add_timer(&tp->timer);
14325
14326         tg3_netif_start(tp);
14327
14328 out:
14329         tg3_full_unlock(tp);
14330
14331         if (!err)
14332                 tg3_phy_start(tp);
14333
14334         return err;
14335 }
14336
14337 static struct pci_driver tg3_driver = {
14338         .name           = DRV_MODULE_NAME,
14339         .id_table       = tg3_pci_tbl,
14340         .probe          = tg3_init_one,
14341         .remove         = __devexit_p(tg3_remove_one),
14342         .suspend        = tg3_suspend,
14343         .resume         = tg3_resume
14344 };
14345
14346 static int __init tg3_init(void)
14347 {
14348         return pci_register_driver(&tg3_driver);
14349 }
14350
14351 static void __exit tg3_cleanup(void)
14352 {
14353         pci_unregister_driver(&tg3_driver);
14354 }
14355
14356 module_init(tg3_init);
14357 module_exit(tg3_cleanup);