/*
 * Broadcom NetXtreme I (BCM57xx / tg3) iPXE driver
 *
 * Covers the BCM57xx family of Gigabit Ethernet controllers found in
 * Intel-based Apple Mac hardware (MacBook, MacBook Pro, iMac, Mac mini).
 *
 * Initialization sequence derived from the Linux tg3 driver:
 *   Copyright (C) 2001, 2002, 2003, 2004 David S. Miller
 *   Copyright (C) 2001, 2002, 2003, 2004 Jeff Garzik
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

FILE_LICENCE ( GPL2_OR_LATER );

#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <byteswap.h>
#include <ipxe/netdevice.h>
#include <ipxe/ethernet.h>
#include <ipxe/if_ether.h>
#include <ipxe/iobuf.h>
#include <ipxe/malloc.h>
#include <ipxe/pci.h>
#include <ipxe/io.h>
#include "tg3.h"

/******************************************************************************
 *
 * Register access helpers
 *
 ******************************************************************************/

static inline uint32_t tg3_read ( struct tg3_nic *tg3, uint32_t reg ) {
	return readl ( tg3->regs + reg );
}

static inline void tg3_write ( struct tg3_nic *tg3, uint32_t reg,
				uint32_t val ) {
	writel ( val, tg3->regs + reg );
}

/** Write to NIC-internal SRAM via the windowed accessor */
static void tg3_nic_sram_write ( struct tg3_nic *tg3, uint32_t addr,
				  uint32_t val ) {
	tg3_write ( tg3, TG3_NIC_SRAM_WIN_BASE, addr );
	tg3_write ( tg3, TG3_NIC_SRAM_WIN_DATA, val );
}


/******************************************************************************
 *
 * NVRAM (flash) access — piece 1: MAC address fallback
 *
 ******************************************************************************/

/**
 * Read one big-endian dword from NVRAM flash
 *
 * Uses the on-chip NVRAM controller.  The chip must be out of reset and the
 * MII clock must be running before calling this.
 */
static int tg3_nvram_read ( struct tg3_nic *tg3, uint32_t offset,
			     uint32_t *val ) {
	unsigned int i;

	/* Enable NVRAM access */
	tg3_write ( tg3, TG3_NVRAM_ACCESS, NVRAM_ACCESS_ENABLE );

	/* Set address and issue a single-word read */
	tg3_write ( tg3, TG3_NVRAM_ADDR, offset );
	tg3_write ( tg3, TG3_NVRAM_CMD,
		     NVRAM_CMD_GO | NVRAM_CMD_RD |
		     NVRAM_CMD_FIRST | NVRAM_CMD_LAST );

	/* Poll for completion */
	for ( i = 0 ; i < 1000 ; i++ ) {
		udelay ( 10 );
		if ( tg3_read ( tg3, TG3_NVRAM_CMD ) & NVRAM_CMD_DONE )
			break;
	}

	/* Disable NVRAM access */
	tg3_write ( tg3, TG3_NVRAM_ACCESS, 0 );

	if ( i >= 1000 ) {
		DBGC ( tg3, "TG3 %p NVRAM read timeout at 0x%x\n", tg3, offset );
		return -ETIMEDOUT;
	}

	/* Data is stored big-endian in flash */
	*val = be32_to_cpu ( tg3_read ( tg3, TG3_NVRAM_RDDATA ) );
	return 0;
}

/**
 * Read the MAC address, falling back to NVRAM if the registers are zeroed
 *
 * Apple's EFI firmware usually populates the MAC registers, so the register
 * path succeeds.  The NVRAM path handles cold-boot cases where firmware has
 * not run yet.
 */
static void tg3_read_mac_addr ( struct tg3_nic *tg3, uint8_t *addr ) {
	uint32_t hi, lo;

	/* Try registers first */
	hi = tg3_read ( tg3, TG3_MAC_ADDR_0_HI );
	lo = tg3_read ( tg3, TG3_MAC_ADDR_0_LO );

	addr[0] = ( hi >> 8 ) & 0xff;
	addr[1] = ( hi >> 0 ) & 0xff;
	addr[2] = ( lo >> 24 ) & 0xff;
	addr[3] = ( lo >> 16 ) & 0xff;
	addr[4] = ( lo >> 8  ) & 0xff;
	addr[5] = ( lo >> 0  ) & 0xff;

	/* Validate: reject all-zeros, all-ones, and multicast first byte */
	if ( ( addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5] ) &&
	     ( addr[0] & addr[1] & addr[2] & addr[3] & addr[4] & addr[5] ) != 0xff &&
	     ! ( addr[0] & 0x01 ) )
		return;

	/* Registers invalid — read from NVRAM (offset 0x7c, big-endian) */
	DBGC ( tg3, "TG3 %p MAC registers empty, trying NVRAM\n", tg3 );
	if ( tg3_nvram_read ( tg3, TG3_NVRAM_MAC_ADDR_OFF + 0, &hi ) != 0 ||
	     tg3_nvram_read ( tg3, TG3_NVRAM_MAC_ADDR_OFF + 4, &lo ) != 0 ) {
		DBGC ( tg3, "TG3 %p NVRAM MAC read failed, MAC may be invalid\n",
		       tg3 );
		return;
	}

	/* NVRAM stores the full 6 bytes in two dwords; low dword is
	 * [byte4][byte5][pad][pad], so shift right 16 to get bytes 4-5. */
	addr[0] = ( hi >> 24 ) & 0xff;
	addr[1] = ( hi >> 16 ) & 0xff;
	addr[2] = ( hi >> 8  ) & 0xff;
	addr[3] = ( hi >> 0  ) & 0xff;
	addr[4] = ( lo >> 24 ) & 0xff;
	addr[5] = ( lo >> 16 ) & 0xff;
}

/* Forward declaration — tg3_mii_read is defined in the MII section below */
static int tg3_mii_read ( struct tg3_nic *tg3, unsigned int reg,
			   uint16_t *val );

/******************************************************************************
 *
 * PHY address probe — piece 2
 *
 ******************************************************************************/

/**
 * Probe for the PHY address on the MII bus
 *
 * Most BCM57xx chips use PHY address 1.  Some (e.g. BCM57766) use address 2.
 * A few chips have an internal PHY that only responds on address 1.
 * Probe addresses 1–7; the first one that returns a non-zero, non-0xffff
 * PHYSID1 wins.
 */
static void tg3_phy_probe ( struct tg3_nic *tg3 ) {
	unsigned int addr;
	uint16_t id1;

	for ( addr = 1 ; addr <= 7 ; addr++ ) {
		tg3->phy_addr = addr;
		if ( tg3_mii_read ( tg3, MII_PHYSID1, &id1 ) != 0 )
			continue;
		if ( id1 == 0 || id1 == 0xffff )
			continue;
		DBGC ( tg3, "TG3 %p PHY at MII addr %u (PHYSID1=%04x)\n",
		       tg3, addr, id1 );
		return;
	}

	/* No PHY found; default to address 1 */
	tg3->phy_addr = 1;
	DBGC ( tg3, "TG3 %p PHY not found, defaulting to addr 1\n", tg3 );
}

/******************************************************************************
 *
 * ASF / management firmware handshake — piece 3
 *
 ******************************************************************************/

/**
 * Notify ASF firmware of driver state change
 *
 * The BCM57xx family has an internal MIPS core running Alert Standard Format
 * (ASF) management firmware.  Without this handshake the firmware may try to
 * use the NIC simultaneously with the driver, causing TX/RX corruption.
 *
 * On chips with APE (Application Processor Engine, e.g. BCM57785) the APE
 * itself manages the handshake; writing to the NIC SRAM mailbox is still
 * harmless and acts as a hint.
 */
static void tg3_asf_handshake ( struct tg3_nic *tg3, uint32_t state ) {
	/* Notify firmware via the driver-state mailbox in NIC SRAM */
	tg3_nic_sram_write ( tg3, TG3_NIC_SRAM_FW_DRV_STATE_MBOX, state );
	/* Small delay to allow the firmware MIPS to observe the write */
	mdelay ( 5 );
	DBGC ( tg3, "TG3 %p ASF handshake: state=0x%08x\n", tg3, state );
}

/******************************************************************************
 *
 * Per-chip-revision quirks — piece 4
 *
 ******************************************************************************/

/**
 * Apply chip-specific configuration adjustments
 *
 * Must be called after tg3_reset() has enabled the memory arbiter but
 * before the buffer manager is enabled, so pool address/size can be set.
 */
static void tg3_chip_quirks ( struct tg3_nic *tg3 ) {
	switch ( tg3->chip_id ) {

	case TG3_DEV_5906:
	case TG3_DEV_5906M:
		/* BCM5906 has only 64 KB of NIC SRAM.  The standard pool
		 * address (0x8000) falls in the middle and leaves no room.
		 * Use a pool that starts after the RCB area (0x300) instead.
		 */
		DBGC ( tg3, "TG3 %p BCM5906: applying small-SRAM quirk\n", tg3 );
		tg3_write ( tg3, TG3_BUFMGR_MB_POOL_ADDR,
			     TG3_MBUF_POOL_ADDR_5906 );
		tg3_write ( tg3, TG3_BUFMGR_MB_POOL_SIZE,
			     TG3_MBUF_POOL_SIZE_5906 );
		break;

	default:
		/* Standard 1 MB SRAM layout — already written in tg3_reset() */
		break;
	}
}

/******************************************************************************
 *
 * MII / PHY interface
 *
 ******************************************************************************/

/** Wait for MII transaction to complete */
static int tg3_mii_wait ( struct tg3_nic *tg3 ) {
	unsigned int i;

	for ( i = 0 ; i < 1000 ; i++ ) {
		if ( ! ( tg3_read ( tg3, TG3_MI_COM ) & MI_COM_BUSY ) )
			return 0;
		udelay ( 10 );
	}
	return -ETIMEDOUT;
}

/** Read a PHY register via MII */
static int tg3_mii_read ( struct tg3_nic *tg3, unsigned int reg,
			   uint16_t *val ) {
	uint32_t cmd;
	int rc;

	cmd = ( MI_COM_START | MI_COM_CMD_READ |
		( tg3->phy_addr << MI_COM_PHY_ADDR_SHIFT ) |
		( reg           << MI_COM_REG_ADDR_SHIFT ) );
	tg3_write ( tg3, TG3_MI_COM, cmd );

	if ( ( rc = tg3_mii_wait ( tg3 ) ) != 0 )
		return rc;

	*val = ( tg3_read ( tg3, TG3_MI_COM ) & MI_COM_DATA_MASK );
	return 0;
}

/** Write a PHY register via MII */
static int tg3_mii_write ( struct tg3_nic *tg3, unsigned int reg,
			    uint16_t val ) {
	uint32_t cmd;

	cmd = ( MI_COM_START | MI_COM_CMD_WRITE |
		( tg3->phy_addr << MI_COM_PHY_ADDR_SHIFT ) |
		( reg           << MI_COM_REG_ADDR_SHIFT ) |
		( val & MI_COM_DATA_MASK ) );
	tg3_write ( tg3, TG3_MI_COM, cmd );

	return tg3_mii_wait ( tg3 );
}

/** Reset and configure the PHY for auto-negotiation */
static int tg3_phy_reset ( struct tg3_nic *tg3 ) {
	unsigned int i;
	uint16_t bmcr;
	int rc;

	/* Software reset */
	if ( ( rc = tg3_mii_write ( tg3, MII_BMCR, BMCR_RESET ) ) != 0 )
		return rc;

	/* Wait for reset to clear */
	for ( i = 0 ; i < 500 ; i++ ) {
		if ( ( rc = tg3_mii_read ( tg3, MII_BMCR, &bmcr ) ) != 0 )
			return rc;
		if ( ! ( bmcr & BMCR_RESET ) )
			break;
		mdelay ( 1 );
	}
	if ( bmcr & BMCR_RESET )
		return -ETIMEDOUT;

	/* Advertise 10/100/1000 full and half duplex */
	if ( ( rc = tg3_mii_write ( tg3, MII_ADVERTISE,
				     ADVERTISE_ALL | ADVERTISE_CSMA |
				     ADVERTISE_PAUSE_CAP ) ) != 0 )
		return rc;

	if ( ( rc = tg3_mii_write ( tg3, MII_CTRL1000,
				     ADVERTISE_1000FULL |
				     ADVERTISE_1000HALF ) ) != 0 )
		return rc;

	/* Restart auto-negotiation */
	return tg3_mii_write ( tg3, MII_BMCR,
				BMCR_ANENABLE | BMCR_ANRESTART );
}

/******************************************************************************
 *
 * Link state
 *
 ******************************************************************************/

static void tg3_check_link ( struct net_device *netdev ) {
	struct tg3_nic *tg3 = netdev->priv;
	uint16_t bmsr;

	if ( tg3_mii_read ( tg3, MII_BMSR, &bmsr ) != 0 )
		return;
	/* Read twice: first read clears latched link-down bit */
	tg3_mii_read ( tg3, MII_BMSR, &bmsr );

	if ( bmsr & BMSR_LSTATUS )
		netdev_link_up ( netdev );
	else
		netdev_link_down ( netdev );
}

/******************************************************************************
 *
 * Hardware reset
 *
 ******************************************************************************/

static int tg3_reset ( struct tg3_nic *tg3 ) {
	uint32_t val;
	uint16_t pci_cmd;
	unsigned int i;

	/* Save PCI command register — the chip reset clears bus-master + memory */
	pci_read_config_word ( tg3->pci, PCI_COMMAND, &pci_cmd );

	/* Chip reset: write CORECLK_RESET to GRC_MISC_CFG (MMIO 0x6804).
	 * MMIO becomes inaccessible for ~120 us while the core clock restarts.
	 */
	tg3_write ( tg3, TG3_GRC_MISC_CFG, GRC_MISC_CFG_CORECLK_RESET );

	/* Flush the posted write, then wait for the core to come back up */
	udelay ( 120 );
	pci_read_config_word ( tg3->pci, PCI_COMMAND, &pci_cmd );
	udelay ( 120 );

	/* Restore PCI command: re-enable bus-master and memory space */
	pci_write_config_word ( tg3->pci, PCI_COMMAND,
				pci_cmd | PCI_COMMAND_MEM | PCI_COMMAND_MASTER );

	/* Give firmware a moment to settle */
	mdelay ( 15 );

	/* Verify MMIO is accessible */
	val = tg3_read ( tg3, TG3_GRC_MODE );
	if ( val == 0xffffffff ) {
		DBGC ( tg3, "TG3 %p MMIO not accessible after reset\n", tg3 );
		return -EIO;
	}

	/* Configure byte/word swap for little-endian host */
	tg3_write ( tg3, TG3_GRC_MODE,
		     GRC_MODE_BSWAP_NONBD | GRC_MODE_WSWAP_NONBD |
		     GRC_MODE_BSWAP_DATA  |
		     GRC_MODE_NO_TX_PHDR_CSUM | GRC_MODE_NO_RX_PHDR_CSUM );

	/* Mask all interrupt sources */
	tg3_write ( tg3, TG3_MBOX_INT_0, 1 );

	/* Enable memory arbiter */
	tg3_write ( tg3, TG3_MEM_ARB_MODE, MEM_ARB_MODE_ENABLE );

	/* Configure buffer manager (defaults; chip quirks may override below) */
	tg3_write ( tg3, TG3_BUFMGR_MB_POOL_ADDR, TG3_MBUF_POOL_ADDR );
	tg3_write ( tg3, TG3_BUFMGR_MB_POOL_SIZE, TG3_MBUF_POOL_SIZE );

	/* Apply per-chip adjustments (e.g. BCM5906 small-SRAM pool) */
	tg3_chip_quirks ( tg3 );

	tg3_write ( tg3, TG3_BUFMGR_MODE,
		     BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE );

	/* Wait for buffer manager ready */
	for ( i = 0 ; i < 1000 ; i++ ) {
		if ( tg3_read ( tg3, TG3_BUFMGR_MODE ) & BUFMGR_MODE_ENABLE )
			break;
		udelay ( 10 );
	}

	/* Enable MII clock */
	tg3_write ( tg3, TG3_MI_MODE, MI_MODE_500KHZ_CONST );
	mdelay ( 40 );

	return 0;
}

/******************************************************************************
 *
 * Ring setup
 *
 ******************************************************************************/

/** Write an RCB (ring control block) to NIC SRAM
 *
 *  Each RCB is 16 bytes: base-addr-hi, base-addr-lo, flags|max-len, nic-addr.
 */
static void tg3_write_rcb ( struct tg3_nic *tg3, uint32_t nic_addr,
			     physaddr_t dma_addr, uint32_t max_len,
			     uint32_t flags ) {
	tg3_nic_sram_write ( tg3, nic_addr + 0,
			     ( ( uint64_t ) dma_addr >> 32 ) );
	tg3_nic_sram_write ( tg3, nic_addr + 4,
			     ( uint32_t ) dma_addr );
	tg3_nic_sram_write ( tg3, nic_addr + 8,
			     ( ( max_len << 16 ) | flags ) );
	tg3_nic_sram_write ( tg3, nic_addr + 12, 0 );
}

static int tg3_alloc_rings ( struct tg3_nic *tg3, struct net_device *netdev ) {
	( void ) netdev;
	unsigned int i;

	/* Status block */
	tg3->status_block = malloc_phys ( TG3_STATBLK_SZ, TG3_STATBLK_ALIGN );
	if ( ! tg3->status_block )
		return -ENOMEM;
	memset ( tg3->status_block, 0, TG3_STATBLK_SZ );

	/* TX ring */
	tg3->tx_ring = malloc_phys ( TG3_TX_RING_SIZE * sizeof ( *tg3->tx_ring ),
				     TG3_TX_RING_SIZE * sizeof ( *tg3->tx_ring ) );
	if ( ! tg3->tx_ring )
		return -ENOMEM;
	memset ( tg3->tx_ring, 0,
		 TG3_TX_RING_SIZE * sizeof ( *tg3->tx_ring ) );

	/* RX standard producer ring */
	tg3->rx_std = malloc_phys ( TG3_RX_STD_RING_SIZE * sizeof ( *tg3->rx_std ),
				    TG3_RX_STD_RING_SIZE * sizeof ( *tg3->rx_std ) );
	if ( ! tg3->rx_std )
		goto err_rx_std;
	memset ( tg3->rx_std, 0,
		 TG3_RX_STD_RING_SIZE * sizeof ( *tg3->rx_std ) );

	/* RX return ring */
	tg3->rx_ret = malloc_phys ( TG3_RX_RET_RING_SIZE * sizeof ( *tg3->rx_ret ),
				    TG3_RX_RET_RING_SIZE * sizeof ( *tg3->rx_ret ) );
	if ( ! tg3->rx_ret )
		goto err_rx_ret;
	memset ( tg3->rx_ret, 0,
		 TG3_RX_RET_RING_SIZE * sizeof ( *tg3->rx_ret ) );

	/* Pre-populate RX producer ring with I/O buffers */
	for ( i = 0 ; i < TG3_RX_STD_RING_SIZE ; i++ ) {
		struct io_buffer *iobuf;
		physaddr_t pa;

		iobuf = alloc_iob ( TG3_RX_STD_MAX_SIZE );
		if ( ! iobuf )
			goto err_alloc_rx;
		tg3->rx_iobuf[i] = iobuf;

		pa = virt_to_phys ( iobuf->data );
		tg3->rx_std[i].addr_hi  = cpu_to_le32 ( ( uint64_t ) pa >> 32 );
		tg3->rx_std[i].addr_lo  = cpu_to_le32 ( ( uint32_t ) pa );
		tg3->rx_std[i].idx_len  = cpu_to_le32 (
			( i << 16 ) | TG3_RX_STD_MAX_SIZE );
		tg3->rx_std[i].opaque   = cpu_to_le32 ( i );
	}
	tg3->rx_std_prod = TG3_RX_STD_RING_SIZE;

	return 0;

 err_alloc_rx:
	for ( i-- ; i < TG3_RX_STD_RING_SIZE ; i-- ) {
		free_iob ( tg3->rx_iobuf[i] );
		tg3->rx_iobuf[i] = NULL;
	}
	free_phys ( tg3->rx_ret,
		    TG3_RX_RET_RING_SIZE * sizeof ( *tg3->rx_ret ) );
 err_rx_ret:
	free_phys ( tg3->rx_std,
		    TG3_RX_STD_RING_SIZE * sizeof ( *tg3->rx_std ) );
 err_rx_std:
	free_phys ( tg3->tx_ring,
		    TG3_TX_RING_SIZE * sizeof ( *tg3->tx_ring ) );
	free_phys ( tg3->status_block, TG3_STATBLK_SZ );
	return -ENOMEM;
}

static void tg3_free_rings ( struct tg3_nic *tg3, struct net_device *netdev ) {
	unsigned int i;

	for ( i = 0 ; i < TG3_TX_RING_SIZE ; i++ ) {
		if ( tg3->tx_iobuf[i] ) {
			netdev_tx_complete_err ( netdev, tg3->tx_iobuf[i],
						 -ECANCELED );
			tg3->tx_iobuf[i] = NULL;
		}
	}
	for ( i = 0 ; i < TG3_RX_STD_RING_SIZE ; i++ ) {
		if ( tg3->rx_iobuf[i] ) {
			free_iob ( tg3->rx_iobuf[i] );
			tg3->rx_iobuf[i] = NULL;
		}
	}

	if ( tg3->rx_ret )
		free_phys ( tg3->rx_ret,
			    TG3_RX_RET_RING_SIZE * sizeof ( *tg3->rx_ret ) );
	if ( tg3->rx_std )
		free_phys ( tg3->rx_std,
			    TG3_RX_STD_RING_SIZE * sizeof ( *tg3->rx_std ) );
	if ( tg3->tx_ring )
		free_phys ( tg3->tx_ring,
			    TG3_TX_RING_SIZE * sizeof ( *tg3->tx_ring ) );
	if ( tg3->status_block )
		free_phys ( tg3->status_block, TG3_STATBLK_SZ );

	tg3->rx_ret       = NULL;
	tg3->rx_std       = NULL;
	tg3->tx_ring      = NULL;
	tg3->status_block = NULL;
}

/******************************************************************************
 *
 * Hardware enable (called from open)
 *
 ******************************************************************************/

static int tg3_hw_init ( struct tg3_nic *tg3, struct net_device *netdev ) {
	physaddr_t pa;
	uint32_t mac_hi, mac_lo;
	int rc;

	/* Reset chip */
	if ( ( rc = tg3_reset ( tg3 ) ) != 0 )
		return rc;

	/* Notify ASF management firmware that the driver is taking over */
	tg3_asf_handshake ( tg3, DRV_STATE_START );

	/* Read MAC address (with NVRAM fallback) */
	tg3_read_mac_addr ( tg3, netdev->hw_addr );

	/* Program MAC address into hardware */
	mac_hi = ( ( (uint32_t) netdev->hw_addr[0] << 8 ) |
		     (uint32_t) netdev->hw_addr[1] );
	mac_lo = ( ( (uint32_t) netdev->hw_addr[2] << 24 ) |
		   ( (uint32_t) netdev->hw_addr[3] << 16 ) |
		   ( (uint32_t) netdev->hw_addr[4] <<  8 ) |
		     (uint32_t) netdev->hw_addr[5] );
	tg3_write ( tg3, TG3_MAC_ADDR_0_HI, mac_hi );
	tg3_write ( tg3, TG3_MAC_ADDR_0_LO, mac_lo );

	/* Clear hash table (accept no multicast) */
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x00, 0 );
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x04, 0 );
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x08, 0 );
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x0c, 0 );
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x10, 0 );
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x14, 0 );
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x18, 0 );
	tg3_write ( tg3, TG3_MAC_HASH_0 + 0x1c, 0 );

	/* Set RX MTU */
	tg3_write ( tg3, TG3_TX_LENGTHS, TX_LENGTHS_DFL );

	/* TX ring RCB → NIC SRAM */
	pa = virt_to_phys ( tg3->tx_ring );
	tg3_write_rcb ( tg3, TG3_NIC_SRAM_SEND_RCB, pa,
			TG3_TX_RING_SIZE, 0 );

	/* RX return ring RCB → NIC SRAM */
	pa = virt_to_phys ( tg3->rx_ret );
	tg3_write_rcb ( tg3, TG3_NIC_SRAM_RCV_RET_RCB, pa,
			TG3_RX_RET_RING_SIZE, 0 );

	/* RX std producer ring info (base addr + ring size) */
	pa = virt_to_phys ( tg3->rx_std );
	tg3_write ( tg3, TG3_STD_RCV_RING_INFO + 0,
		     ( uint32_t ) ( ( uint64_t ) pa >> 32 ) );
	tg3_write ( tg3, TG3_STD_RCV_RING_INFO + 4, ( uint32_t ) pa );
	tg3_write ( tg3, TG3_STD_RCV_RING_INFO + 8,
		     ( TG3_RX_STD_RING_SIZE << 16 ) );

	/* Set up host coalescing: point NIC at the status block DMA buffer.
	 * Use one-frame coalescing so every TX/RX triggers a status update.
	 */
	{
		physaddr_t sb_pa = virt_to_phys ( tg3->status_block );
		tg3_write ( tg3, TG3_HOSTCC_RXCOL_TICKS,  0 );
		tg3_write ( tg3, TG3_HOSTCC_TXCOL_TICKS,  0 );
		tg3_write ( tg3, TG3_HOSTCC_RXMAX_FRAMES, 1 );
		tg3_write ( tg3, TG3_HOSTCC_TXMAX_FRAMES, 1 );
		tg3_write ( tg3, TG3_HOSTCC_STATBLK_HI,
			     ( uint32_t ) ( ( uint64_t ) sb_pa >> 32 ) );
		tg3_write ( tg3, TG3_HOSTCC_STATBLK_LO,
			     ( uint32_t ) sb_pa );
	}
	tg3_write ( tg3, TG3_HOSTCC_MODE, HOSTCC_MODE_ENABLE );

	/* Enable DMA engines */
	tg3_write ( tg3, TG3_RDMA_MODE, RDMA_MODE_ENABLE | RDMA_MODE_FIFO_LONG_BURST );
	tg3_write ( tg3, TG3_WDMA_MODE, WDMA_MODE_ENABLE );

	/* Enable send path subsystems */
	tg3_write ( tg3, TG3_SNDBDC_MODE,  SNDBDC_MODE_ENABLE );
	tg3_write ( tg3, TG3_SNDBDS_MODE,  SNDBDS_MODE_ENABLE );
	tg3_write ( tg3, TG3_SNDDATA_MODE, SNDDATA_MODE_ENABLE );
	tg3_write ( tg3, TG3_SNDBDI_MODE,  SNDBDI_MODE_ENABLE );

	/* Enable receive path subsystems */
	tg3_write ( tg3, TG3_RCVBDI_MODE,  RCVBDI_MODE_ENABLE );
	tg3_write ( tg3, TG3_RCVDBDI_MODE, RCVDBDI_MODE_ENABLE );
	tg3_write ( tg3, TG3_RCVCC_MODE,   RCVCC_MODE_ENABLE );

	/* Enable BD completion engines */
	tg3_write ( tg3, TG3_SBDC_MODE, SBDC_MODE_ENABLE );
	tg3_write ( tg3, TG3_RBDC_MODE, RBDC_MODE_ENABLE );

	/* Enable TX MAC */
	tg3_write ( tg3, TG3_TX_MODE, TX_MODE_ENABLE );

	/* Enable RX MAC */
	tg3_write ( tg3, TG3_RX_MODE, RX_MODE_ENABLE | RX_MODE_KEEP_VLAN_TAG );

	/* Enable MAC datapath */
	tg3_write ( tg3, TG3_MAC_MODE,
		     MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE |
		     MAC_MODE_FHDE_ENABLE |
		     MAC_MODE_RXSTAT_ENABLE | MAC_MODE_TXSTAT_ENABLE );

	/* Tell NIC about the initial RX std producer index */
	tg3_write ( tg3, TG3_MBOX_RCV_STD_PROD,
		     tg3->rx_std_prod % TG3_RX_STD_RING_SIZE );

	/* Reset TX/RX consumer/producer indices */
	tg3->tx_prod = 0;
	tg3->tx_cons = 0;
	tg3->rx_ret_cons = 0;

	/* Probe PHY bus address, then reset and configure the PHY */
	tg3_phy_probe ( tg3 );
	if ( ( rc = tg3_phy_reset ( tg3 ) ) != 0 ) {
		DBGC ( tg3, "TG3 %p PHY reset failed: %s\n", tg3, strerror ( rc ) );
		/* Non-fatal: continue without PHY reset */
	}

	return 0;
}

/******************************************************************************
 *
 * Network device operations
 *
 ******************************************************************************/

static int tg3_open ( struct net_device *netdev ) {
	struct tg3_nic *tg3 = netdev->priv;
	int rc;

	if ( ( rc = tg3_alloc_rings ( tg3, netdev ) ) != 0 )
		return rc;

	if ( ( rc = tg3_hw_init ( tg3, netdev ) ) != 0 ) {
		tg3_free_rings ( tg3, netdev );
		return rc;
	}

	return 0;
}

static void tg3_close ( struct net_device *netdev ) {
	struct tg3_nic *tg3 = netdev->priv;

	/* Disable MAC datapath */
	tg3_write ( tg3, TG3_MAC_MODE, 0 );
	tg3_write ( tg3, TG3_TX_MODE, 0 );
	tg3_write ( tg3, TG3_RX_MODE, 0 );

	/* Notify ASF firmware that driver is releasing the NIC */
	tg3_asf_handshake ( tg3, DRV_STATE_UNLOAD );

	tg3_free_rings ( tg3, netdev );
}

static int tg3_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
	struct tg3_nic *tg3 = netdev->priv;
	unsigned int slot;
	physaddr_t pa;
	uint32_t len_flags;

	/* Check ring is not full */
	if ( ( tg3->tx_prod - tg3->tx_cons ) >= TG3_TX_RING_SIZE )
		return -ENOBUFS;

	slot = tg3->tx_prod % TG3_TX_RING_SIZE;
	pa   = virt_to_phys ( iobuf->data );

	len_flags = ( ( iob_len ( iobuf ) << TG3_TX_BD_LEN_SHIFT ) |
		      TG3_TX_BD_FLAG_END );

	tg3->tx_ring[slot].addr_hi  = cpu_to_le32 ( ( uint64_t ) pa >> 32 );
	tg3->tx_ring[slot].addr_lo  = cpu_to_le32 ( ( uint32_t ) pa );
	tg3->tx_ring[slot].len_flags = cpu_to_le32 ( len_flags );
	tg3->tx_ring[slot].vlan_tag  = 0;
	wmb();

	tg3->tx_iobuf[slot] = iobuf;
	tg3->tx_prod++;

	/* Kick TX */
	tg3_write ( tg3, TG3_MBOX_SND_PROD_0,
		     tg3->tx_prod % TG3_TX_RING_SIZE );

	return 0;
}

static void tg3_poll_tx ( struct net_device *netdev ) {
	struct tg3_nic *tg3 = netdev->priv;
	unsigned int cons;

	/* Force NIC to flush consumer indices to the status block */
	tg3_write ( tg3, TG3_HOSTCC_MODE, HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW );
	rmb();

	/* TX consumer index is in the per-queue idx[0].tx_consumer field */
	cons = le16_to_cpu ( tg3->status_block->idx[0].tx_consumer );

	while ( tg3->tx_cons != cons ) {
		unsigned int slot = tg3->tx_cons % TG3_TX_RING_SIZE;
		struct io_buffer *iobuf = tg3->tx_iobuf[slot];

		if ( iobuf ) {
			netdev_tx_complete ( netdev, iobuf );
			tg3->tx_iobuf[slot] = NULL;
		}
		tg3->tx_cons++;
	}
}

static void tg3_poll_rx ( struct net_device *netdev ) {
	struct tg3_nic *tg3 = netdev->priv;
	struct tg3_rx_ret_bd *ret;
	struct io_buffer *iobuf;
	unsigned int cons, slot;
	uint32_t flags, len;
	int rc;

	/* RX return consumer written by NIC to status_block->idx[0].rx_consumer */
	rmb();
	cons = le16_to_cpu ( tg3->status_block->idx[0].rx_consumer );

	while ( tg3->rx_ret_cons != cons ) {
		ret   = &tg3->rx_ret[tg3->rx_ret_cons % TG3_RX_RET_RING_SIZE];
		flags = le32_to_cpu ( ret->flags );
		len   = le32_to_cpu ( ret->idx_len ) & 0xffff;
		slot  = ( le32_to_cpu ( ret->opaque ) ) & ( TG3_RX_STD_RING_SIZE - 1 );

		iobuf = tg3->rx_iobuf[slot];
		tg3->rx_iobuf[slot] = NULL;

		if ( flags & TG3_RX_RET_FLAG_ERROR ) {
			rc = -EIO;
			netdev_rx_err ( netdev, iobuf, rc );
		} else {
			iob_put ( iobuf, len );
			netdev_rx ( netdev, iobuf );
		}

		/* Refill this slot */
		iobuf = alloc_iob ( TG3_RX_STD_MAX_SIZE );
		if ( iobuf ) {
			physaddr_t pa = virt_to_phys ( iobuf->data );

			tg3->rx_iobuf[slot] = iobuf;
			tg3->rx_std[slot].addr_hi = cpu_to_le32 ( ( uint64_t ) pa >> 32 );
			tg3->rx_std[slot].addr_lo = cpu_to_le32 ( ( uint32_t ) pa );
			tg3->rx_std[slot].idx_len = cpu_to_le32 (
				( slot << 16 ) | TG3_RX_STD_MAX_SIZE );
			tg3->rx_std[slot].opaque  = cpu_to_le32 ( slot );
			wmb();

			tg3->rx_std_prod++;
			tg3_write ( tg3, TG3_MBOX_RCV_STD_PROD,
				     tg3->rx_std_prod % TG3_RX_STD_RING_SIZE );
		}

		tg3->rx_ret_cons++;
		tg3_write ( tg3, TG3_MBOX_RCV_RET_CON_0,
			     tg3->rx_ret_cons % TG3_RX_RET_RING_SIZE );
	}
}

static void tg3_poll ( struct net_device *netdev ) {
	tg3_poll_tx ( netdev );
	tg3_poll_rx ( netdev );
	tg3_check_link ( netdev );
}

static struct net_device_operations tg3_operations = {
	.open     = tg3_open,
	.close    = tg3_close,
	.transmit = tg3_transmit,
	.poll     = tg3_poll,
};

/******************************************************************************
 *
 * PCI probe / remove
 *
 ******************************************************************************/

static int tg3_probe ( struct pci_device *pci ) {
	struct net_device *netdev;
	struct tg3_nic *tg3;
	int rc;

	netdev = alloc_etherdev ( sizeof ( *tg3 ) );
	if ( ! netdev )
		return -ENOMEM;

	netdev_init ( netdev, &tg3_operations );
	tg3 = netdev->priv;
	pci_set_drvdata ( pci, netdev );
	netdev->dev = &pci->dev;
	memset ( tg3, 0, sizeof ( *tg3 ) );
	tg3->pci     = pci;
	tg3->chip_id = pci->device;

	/* Map MMIO registers (BAR 0) */
	adjust_pci_device ( pci );
	tg3->regs = ioremap ( pci->membase, 0x8000 );
	if ( ! tg3->regs ) {
		rc = -ENODEV;
		goto err_ioremap;
	}

	/* Read MAC address (with NVRAM fallback for cold-boot cases) */
	tg3_read_mac_addr ( tg3, netdev->hw_addr );

	if ( ( rc = register_netdev ( netdev ) ) != 0 )
		goto err_register;

	DBGC ( tg3, "TG3 %p at %p (IRQ %d) MAC %s\n", tg3, tg3->regs,
	       pci->irq, eth_ntoa ( netdev->hw_addr ) );

	return 0;

 err_register:
	iounmap ( tg3->regs );
 err_ioremap:
	netdev_nullify ( netdev );
	netdev_put ( netdev );
	return rc;
}

static void tg3_remove ( struct pci_device *pci ) {
	struct net_device *netdev = pci_get_drvdata ( pci );
	struct tg3_nic *tg3 = netdev->priv;

	unregister_netdev ( netdev );
	iounmap ( tg3->regs );
	netdev_nullify ( netdev );
	netdev_put ( netdev );
}

static struct pci_device_id tg3_nics[] = {
	PCI_ROM ( 0x14e4, 0x1677, "bcm5751",  "Broadcom BCM5751",  0 ),
	PCI_ROM ( 0x14e4, 0x167d, "bcm5751m", "Broadcom BCM5751M", 0 ),
	PCI_ROM ( 0x14e4, 0x167a, "bcm5754",  "Broadcom BCM5754",  0 ),
	PCI_ROM ( 0x14e4, 0x1672, "bcm5754m", "Broadcom BCM5754M", 0 ),
	PCI_ROM ( 0x14e4, 0x167b, "bcm5755",  "Broadcom BCM5755",  0 ),
	PCI_ROM ( 0x14e4, 0x169c, "bcm5755m", "Broadcom BCM5755M", 0 ),
	PCI_ROM ( 0x14e4, 0x1674, "bcm5756m", "Broadcom BCM5756M", 0 ),
	PCI_ROM ( 0x14e4, 0x1680, "bcm5761",  "Broadcom BCM5761",  0 ),
	PCI_ROM ( 0x14e4, 0x1681, "bcm5761e", "Broadcom BCM5761E", 0 ),
	PCI_ROM ( 0x14e4, 0x1688, "bcm5761s", "Broadcom BCM5761S", 0 ),
	PCI_ROM ( 0x14e4, 0x1689, "bcm5761se","Broadcom BCM5761SE",0 ),
	PCI_ROM ( 0x14e4, 0x1684, "bcm5764m", "Broadcom BCM5764M", 0 ),
	PCI_ROM ( 0x14e4, 0x1698, "bcm5784",  "Broadcom BCM5784",  0 ),
	PCI_ROM ( 0x14e4, 0x169b, "bcm5787",  "Broadcom BCM5787",  0 ),
	PCI_ROM ( 0x14e4, 0x1693, "bcm5787m", "Broadcom BCM5787M", 0 ),
	PCI_ROM ( 0x14e4, 0x1712, "bcm5906",  "Broadcom BCM5906",  0 ),
	PCI_ROM ( 0x14e4, 0x1713, "bcm5906m", "Broadcom BCM5906M", 0 ),
	PCI_ROM ( 0x14e4, 0x16b4, "bcm57780", "Broadcom BCM57780", 0 ),
	PCI_ROM ( 0x14e4, 0x16b7, "bcm57781", "Broadcom BCM57781", 0 ),
	PCI_ROM ( 0x14e4, 0x16b5, "bcm57785", "Broadcom BCM57785", 0 ),
	PCI_ROM ( 0x14e4, 0x16b6, "bcm57788", "Broadcom BCM57788", 0 ),
	PCI_ROM ( 0x14e4, 0x1682, "bcm57762", "Broadcom BCM57762", 0 ),
	PCI_ROM ( 0x14e4, 0x1686, "bcm57766", "Broadcom BCM57766", 0 ),
};

struct pci_driver tg3_driver __pci_driver = {
	.ids     = tg3_nics,
	.id_count = ( sizeof ( tg3_nics ) / sizeof ( tg3_nics[0] ) ),
	.probe   = tg3_probe,
	.remove  = tg3_remove,
};
