xref: /illumos-gate/usr/src/uts/common/io/sfxge/sfxge_tx.c (revision 49ef7e0638c8b771d8a136eae78b1c0f99acc8e0)

/*
 * Copyright (c) 2008-2016 Solarflare Communications Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/atomic.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/strsubr.h>
#include <sys/pattr.h>
#include <sys/cpu.h>

#include <sys/ethernet.h>
#include <inet/ip.h>

#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>

#include "sfxge.h"

#include "efx.h"

/* TXQ flush response timeout (in microseconds) */
#define	SFXGE_TX_QFLUSH_USEC	(2000000)

/* See sfxge.conf.private for descriptions */
#define	SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT 4096
#define	SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT 256


/* Transmit buffer DMA attributes */
static ddi_device_acc_attr_t sfxge_tx_buffer_devacc = {

	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
};

static ddi_dma_attr_t sfxge_tx_buffer_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version	*/
	0,			/* dma_attr_addr_lo	*/
	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
	0xffffffffffffffffull,	/* dma_attr_count_max	*/
	SFXGE_TX_BUFFER_SIZE,	/* dma_attr_align	*/
	0xffffffff,		/* dma_attr_burstsizes	*/
	1,			/* dma_attr_minxfer	*/
	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
	0xffffffffffffffffull,	/* dma_attr_seg		*/
	1,			/* dma_attr_sgllen	*/
	1,			/* dma_attr_granular	*/
	0			/* dma_attr_flags	*/
};

/* Transmit mapping DMA attributes */
static ddi_dma_attr_t sfxge_tx_mapping_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version	*/
	0,			/* dma_attr_addr_lo	*/
	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
	0xffffffffffffffffull,	/* dma_attr_count_max	*/
	1,			/* dma_attr_align	*/
	0xffffffff,		/* dma_attr_burstsizes	*/
	1,			/* dma_attr_minxfer	*/
	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
	0xffffffffffffffffull,	/* dma_attr_seg		*/
	0x7fffffff,		/* dma_attr_sgllen	*/
	1,			/* dma_attr_granular	*/
	0			/* dma_attr_flags	*/
};

/* Transmit queue DMA attributes */
static ddi_device_acc_attr_t sfxge_txq_devacc = {

	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
};

static ddi_dma_attr_t sfxge_txq_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version	*/
	0,			/* dma_attr_addr_lo	*/
	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
	0xffffffffffffffffull,	/* dma_attr_count_max	*/
	EFX_BUF_SIZE,		/* dma_attr_align	*/
	0xffffffff,		/* dma_attr_burstsizes	*/
	1,			/* dma_attr_minxfer	*/
	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
	0xffffffffffffffffull,	/* dma_attr_seg		*/
	1,			/* dma_attr_sgllen	*/
	1,			/* dma_attr_granular	*/
	0			/* dma_attr_flags	*/
};


/*
 * A sfxge_tx_qdpl_swizzle() can happen when the DPL get list is one packet
 * under the limit, and must move all packets from the DPL put->get list
 * Hence this is the real maximum length of the TX DPL get list.
 */
static int
sfxge_tx_dpl_get_pkt_max(sfxge_txq_t *stp)
{
	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
	return (stdp->get_pkt_limit + stdp->put_pkt_limit - 1);
}


static int
sfxge_tx_packet_ctor(void *buf, void *arg, int kmflags)
{
	_NOTE(ARGUNUSED(arg, kmflags))

	bzero(buf, sizeof (sfxge_tx_packet_t));

	return (0);
}

static void
sfxge_tx_packet_dtor(void *buf, void *arg)
{
	sfxge_tx_packet_t *stpp = buf;

	_NOTE(ARGUNUSED(arg))

	SFXGE_OBJ_CHECK(stpp, sfxge_tx_packet_t);
}

static int
sfxge_tx_buffer_ctor(void *buf, void *arg, int kmflags)
{
	sfxge_tx_buffer_t *stbp = buf;
	sfxge_t *sp = arg;
	sfxge_dma_buffer_attr_t dma_attr;
	int rc;

	bzero(buf, sizeof (sfxge_tx_buffer_t));

	dma_attr.sdba_dip	 = sp->s_dip;
	dma_attr.sdba_dattrp	 = &sfxge_tx_buffer_dma_attr;
	dma_attr.sdba_callback	 = ((kmflags == KM_SLEEP) ?
	    DDI_DMA_SLEEP : DDI_DMA_DONTWAIT);
	dma_attr.sdba_length	 = SFXGE_TX_BUFFER_SIZE;
	dma_attr.sdba_memflags	 = DDI_DMA_STREAMING;
	dma_attr.sdba_devaccp	 = &sfxge_tx_buffer_devacc;
	dma_attr.sdba_bindflags	 = DDI_DMA_WRITE | DDI_DMA_STREAMING;
	dma_attr.sdba_maxcookies = 1;
	dma_attr.sdba_zeroinit	 = B_FALSE;

	if ((rc = sfxge_dma_buffer_create(&(stbp->stb_esm), &dma_attr)) != 0)
		goto fail1;

	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	SFXGE_OBJ_CHECK(stbp, sfxge_tx_buffer_t);

	return (-1);
}

static void
sfxge_tx_buffer_dtor(void *buf, void *arg)
{
	sfxge_tx_buffer_t *stbp = buf;

	_NOTE(ARGUNUSED(arg))

	sfxge_dma_buffer_destroy(&(stbp->stb_esm));

	SFXGE_OBJ_CHECK(stbp, sfxge_tx_buffer_t);
}

static int
sfxge_tx_mapping_ctor(void *buf, void *arg, int kmflags)
{
	sfxge_tx_mapping_t *stmp = buf;
	sfxge_t *sp = arg;
	dev_info_t *dip = sp->s_dip;
	int rc;

	bzero(buf, sizeof (sfxge_tx_mapping_t));

	stmp->stm_sp = sp;

	/* Allocate DMA handle */
	rc = ddi_dma_alloc_handle(dip, &sfxge_tx_mapping_dma_attr,
	    (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
	    NULL, &(stmp->stm_dma_handle));
	if (rc != DDI_SUCCESS)
		goto fail1;

	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	stmp->stm_sp = NULL;

	SFXGE_OBJ_CHECK(stmp, sfxge_tx_mapping_t);

	return (-1);
}

static void
sfxge_tx_mapping_dtor(void *buf, void *arg)
{
	sfxge_tx_mapping_t *stmp = buf;

	ASSERT3P(stmp->stm_sp, ==, arg);

	/* Free the DMA handle */
	ddi_dma_free_handle(&(stmp->stm_dma_handle));
	stmp->stm_dma_handle = NULL;

	stmp->stm_sp = NULL;

	SFXGE_OBJ_CHECK(stmp, sfxge_tx_mapping_t);
}

static int
sfxge_tx_qctor(void *buf, void *arg, int kmflags)
{
	sfxge_txq_t *stp = buf;
	efsys_mem_t *esmp = &(stp->st_mem);
	sfxge_t *sp = arg;
	sfxge_dma_buffer_attr_t dma_attr;
	sfxge_tx_dpl_t *stdp;
	int rc;

	/* Compile-time structure layout checks */
	EFX_STATIC_ASSERT(sizeof (stp->__st_u1.__st_s1) <=
	    sizeof (stp->__st_u1.__st_pad));
	EFX_STATIC_ASSERT(sizeof (stp->__st_u2.__st_s2) <=
	    sizeof (stp->__st_u2.__st_pad));
	EFX_STATIC_ASSERT(sizeof (stp->__st_u3.__st_s3) <=
	    sizeof (stp->__st_u3.__st_pad));
	EFX_STATIC_ASSERT(sizeof (stp->__st_u4.__st_s4) <=
	    sizeof (stp->__st_u4.__st_pad));

	bzero(buf, sizeof (sfxge_txq_t));

	stp->st_sp = sp;

	dma_attr.sdba_dip	 = sp->s_dip;
	dma_attr.sdba_dattrp	 = &sfxge_txq_dma_attr;
	dma_attr.sdba_callback	 = DDI_DMA_SLEEP;
	dma_attr.sdba_length	 = EFX_TXQ_SIZE(SFXGE_TX_NDESCS);
	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
	dma_attr.sdba_devaccp	 = &sfxge_txq_devacc;
	dma_attr.sdba_bindflags	 = DDI_DMA_READ | DDI_DMA_CONSISTENT;
	dma_attr.sdba_maxcookies = EFX_TXQ_NBUFS(SFXGE_TX_NDESCS);
	dma_attr.sdba_zeroinit	 = B_FALSE;

	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
		goto fail1;

	/* Allocate some buffer table entries */
	if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS),
	    &(stp->st_id))) != 0)
		goto fail2;

	/* Allocate the descriptor array */
	if ((stp->st_eb = kmem_zalloc(sizeof (efx_buffer_t) *
	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS), kmflags)) == NULL) {
		rc = ENOMEM;
		goto fail3;
	}

	/* Allocate the context arrays */
	if ((stp->st_stmp = kmem_zalloc(sizeof (sfxge_tx_mapping_t *) *
	    SFXGE_TX_NDESCS, kmflags)) == NULL) {
		rc = ENOMEM;
		goto fail4;
	}

	if ((stp->st_stbp = kmem_zalloc(sizeof (sfxge_tx_buffer_t *) *
	    SFXGE_TX_NDESCS, kmflags)) == NULL) {
		rc = ENOMEM;
		goto fail5;
	}

	if ((stp->st_mp = kmem_zalloc(sizeof (mblk_t *) *
	    SFXGE_TX_NDESCS, kmflags)) == NULL) {
		rc = ENOMEM;
		goto fail6;
	}

	/* Initialize the deferred packet list */
	stdp = &(stp->st_dpl);
	stdp->std_getp = &(stdp->std_get);

	stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;

	return (0);

fail6:
	DTRACE_PROBE(fail6);

	kmem_free(stp->st_stbp, sizeof (sfxge_tx_buffer_t *) * SFXGE_TX_NDESCS);
	stp->st_stbp = NULL;

fail5:
	DTRACE_PROBE(fail5);

	kmem_free(stp->st_stmp,
	    sizeof (sfxge_tx_mapping_t *) * SFXGE_TX_NDESCS);
	stp->st_stmp = NULL;

fail4:
	DTRACE_PROBE(fail4);

	/* Free the descriptor array */
	kmem_free(stp->st_eb, sizeof (efx_buffer_t) *
	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
	stp->st_eb = NULL;

fail3:
	DTRACE_PROBE(fail3);

	/* Free the buffer table entries */
	sfxge_sram_buf_tbl_free(sp, stp->st_id, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));
	stp->st_id = 0;

fail2:
	DTRACE_PROBE(fail2);

	/* Tear down DMA setup */
	sfxge_dma_buffer_destroy(esmp);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	stp->st_sp = NULL;

	SFXGE_OBJ_CHECK(stp, sfxge_txq_t);

	return (-1);
}

static void
sfxge_tx_qdtor(void *buf, void *arg)
{
	sfxge_txq_t *stp = buf;
	efsys_mem_t *esmp = &(stp->st_mem);
	sfxge_t *sp = stp->st_sp;
	sfxge_tx_dpl_t *stdp;

	_NOTE(ARGUNUSED(arg))

	stp->st_unblock = 0;

	/* Tear down the deferred packet list */
	stdp = &(stp->st_dpl);
	ASSERT3P(stdp->std_getp, ==, &(stdp->std_get));
	stdp->std_getp = NULL;

	/* Free the context arrays */
	kmem_free(stp->st_mp, sizeof (mblk_t *) * SFXGE_TX_NDESCS);
	stp->st_mp = NULL;

	kmem_free(stp->st_stbp, sizeof (sfxge_tx_buffer_t *) * SFXGE_TX_NDESCS);
	stp->st_stbp = NULL;

	kmem_free(stp->st_stmp,
	    sizeof (sfxge_tx_mapping_t *) * SFXGE_TX_NDESCS);
	stp->st_stmp = NULL;

	/* Free the descriptor array */
	kmem_free(stp->st_eb, sizeof (efx_buffer_t) *
	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
	stp->st_eb = NULL;

	/* Free the buffer table entries */
	sfxge_sram_buf_tbl_free(sp, stp->st_id, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));
	stp->st_id = 0;

	/* Tear down dma setup */
	sfxge_dma_buffer_destroy(esmp);

	stp->st_sp = NULL;

	SFXGE_OBJ_CHECK(stp, sfxge_txq_t);
}

static void
sfxge_tx_packet_destroy(sfxge_t *sp, sfxge_tx_packet_t *stpp)
{
	kmem_cache_free(sp->s_tpc, stpp);
}

static sfxge_tx_packet_t *
sfxge_tx_packet_create(sfxge_t *sp)
{
	sfxge_tx_packet_t *stpp;

	stpp = kmem_cache_alloc(sp->s_tpc, KM_NOSLEEP);

	return (stpp);
}

static inline int
sfxge_tx_qfpp_put(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp)
{
	sfxge_tx_fpp_t *stfp = &(stp->st_fpp);

	ASSERT(mutex_owned(&(stp->st_lock)));

	ASSERT3P(stpp->stp_next, ==, NULL);
	ASSERT3P(stpp->stp_mp, ==, NULL);
	ASSERT3P(stpp->stp_etherhp, ==, NULL);
	ASSERT3P(stpp->stp_iphp, ==, NULL);
	ASSERT3P(stpp->stp_thp, ==, NULL);
	ASSERT3U(stpp->stp_off, ==, 0);
	ASSERT3U(stpp->stp_size, ==, 0);
	ASSERT3U(stpp->stp_mss, ==, 0);
	ASSERT3U(stpp->stp_dpl_put_len, ==, 0);

	if (stfp->stf_count < SFXGE_TX_FPP_MAX) {
		/* Add to the start of the list */
		stpp->stp_next = stfp->stf_stpp;
		stfp->stf_stpp = stpp;
		stfp->stf_count++;

		return (0);
	}

	DTRACE_PROBE(fpp_full);
	return (ENOSPC);
}

static inline sfxge_tx_packet_t *
sfxge_tx_qfpp_get(sfxge_txq_t *stp)
{
	sfxge_tx_packet_t *stpp;
	sfxge_tx_fpp_t *stfp = &(stp->st_fpp);

	ASSERT(mutex_owned(&(stp->st_lock)));

	stpp = stfp->stf_stpp;
	if (stpp == NULL) {
		ASSERT3U(stfp->stf_count, ==, 0);
		return (NULL);
	}

	/* Remove item from the head of the list */
	stfp->stf_stpp = stpp->stp_next;
	stpp->stp_next = NULL;

	ASSERT3U(stfp->stf_count, >, 0);
	stfp->stf_count--;

	if (stfp->stf_count != 0) {
		ASSERT(stfp->stf_stpp != NULL);
		prefetch_read_many(stfp->stf_stpp);
	}
	return (stpp);
}

static void
sfxge_tx_qfpp_empty(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	sfxge_tx_fpp_t *stfp = &(stp->st_fpp);
	sfxge_tx_packet_t *stpp;

	mutex_enter(&(stp->st_lock));

	stpp = stfp->stf_stpp;
	stfp->stf_stpp = NULL;

	while (stpp != NULL) {
		sfxge_tx_packet_t *next;

		next = stpp->stp_next;
		stpp->stp_next = NULL;

		ASSERT3U(stfp->stf_count, >, 0);
		stfp->stf_count--;

		sfxge_tx_packet_destroy(sp, stpp);

		stpp = next;
	}
	ASSERT3U(stfp->stf_count, ==, 0);

	mutex_exit(&(stp->st_lock));
}

static inline void
sfxge_tx_qfbp_put(sfxge_txq_t *stp, sfxge_tx_buffer_t *stbp)
{
	sfxge_tx_fbp_t *stfp = &(stp->st_fbp);

	ASSERT3P(stbp->stb_next, ==, NULL);
	ASSERT3U(stbp->stb_off, ==, 0);
	ASSERT3U(stbp->stb_esm.esm_used, ==, 0);

	stbp->stb_next = stfp->stf_stbp;
	stfp->stf_stbp = stbp;
	stfp->stf_count++;
}


static inline sfxge_tx_buffer_t *
sfxge_tx_qfbp_get(sfxge_txq_t *stp)
{
	sfxge_tx_buffer_t *stbp;
	sfxge_tx_fbp_t *stfp = &(stp->st_fbp);

	stbp = stfp->stf_stbp;
	if (stbp == NULL) {
		ASSERT3U(stfp->stf_count, ==, 0);
		return (NULL);
	}

	stfp->stf_stbp = stbp->stb_next;
	stbp->stb_next = NULL;

	ASSERT3U(stfp->stf_count, >, 0);
	stfp->stf_count--;

	if (stfp->stf_count != 0) {
		ASSERT(stfp->stf_stbp != NULL);
		prefetch_read_many(stfp->stf_stbp);
	}

	return (stbp);
}

static void
sfxge_tx_qfbp_empty(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	sfxge_tx_fbp_t *stfp = &(stp->st_fbp);
	sfxge_tx_buffer_t *stbp;

	mutex_enter(&(stp->st_lock));

	stbp = stfp->stf_stbp;
	stfp->stf_stbp = NULL;

	while (stbp != NULL) {
		sfxge_tx_buffer_t *next;

		next = stbp->stb_next;
		stbp->stb_next = NULL;

		ASSERT3U(stfp->stf_count, >, 0);
		stfp->stf_count--;

		kmem_cache_free(sp->s_tbc, stbp);

		stbp = next;
	}
	ASSERT3U(stfp->stf_count, ==, 0);

	mutex_exit(&(stp->st_lock));
}

static inline void
sfxge_tx_qfmp_put(sfxge_txq_t *stp, sfxge_tx_mapping_t *stmp)
{
	sfxge_tx_fmp_t *stfp = &(stp->st_fmp);

	ASSERT3P(stmp->stm_next, ==, NULL);
	ASSERT3P(stmp->stm_mp, ==, NULL);
	ASSERT3P(stmp->stm_base, ==, NULL);
	ASSERT3U(stmp->stm_off, ==, 0);
	ASSERT3U(stmp->stm_size, ==, 0);

	stmp->stm_next = stfp->stf_stmp;
	stfp->stf_stmp = stmp;
	stfp->stf_count++;
}

static inline sfxge_tx_mapping_t *
sfxge_tx_qfmp_get(sfxge_txq_t *stp)
{
	sfxge_tx_mapping_t *stmp;
	sfxge_tx_fmp_t *stfp = &(stp->st_fmp);

	stmp = stfp->stf_stmp;
	if (stmp == NULL) {
		ASSERT3U(stfp->stf_count, ==, 0);
		return (NULL);
	}

	stfp->stf_stmp = stmp->stm_next;
	stmp->stm_next = NULL;

	ASSERT3U(stfp->stf_count, >, 0);
	stfp->stf_count--;

	if (stfp->stf_count != 0) {
		ASSERT(stfp->stf_stmp != NULL);
		prefetch_read_many(stfp->stf_stmp);
	}
	return (stmp);
}

static void
sfxge_tx_qfmp_empty(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	sfxge_tx_fmp_t *stfp = &(stp->st_fmp);
	sfxge_tx_mapping_t *stmp;

	mutex_enter(&(stp->st_lock));

	stmp = stfp->stf_stmp;
	stfp->stf_stmp = NULL;

	while (stmp != NULL) {
		sfxge_tx_mapping_t *next;

		next = stmp->stm_next;
		stmp->stm_next = NULL;

		ASSERT3U(stfp->stf_count, >, 0);
		stfp->stf_count--;

		kmem_cache_free(sp->s_tmc, stmp);

		stmp = next;
	}
	ASSERT3U(stfp->stf_count, ==, 0);

	mutex_exit(&(stp->st_lock));
}

static void
sfxge_tx_msgb_unbind(sfxge_tx_mapping_t *stmp)
{
	bzero(stmp->stm_addr, sizeof (uint64_t) * SFXGE_TX_MAPPING_NADDR);
	stmp->stm_off = 0;

	(void) ddi_dma_unbind_handle(stmp->stm_dma_handle);

	stmp->stm_size = 0;
	stmp->stm_base = NULL;

	stmp->stm_mp = NULL;
}

#define	SFXGE_TX_DESCSHIFT	12
#define	SFXGE_TX_DESCSIZE	(1 << 12)

#define	SFXGE_TX_DESCOFFSET	(SFXGE_TX_DESCSIZE - 1)
#define	SFXGE_TX_DESCMASK	(~SFXGE_TX_DESCOFFSET)

static int
sfxge_tx_msgb_bind(mblk_t *mp, sfxge_tx_mapping_t *stmp)
{
	ddi_dma_cookie_t dmac;
	unsigned int ncookies;
	size_t size;
	unsigned int n;
	int rc;

	ASSERT(mp != NULL);
	ASSERT3U(DB_TYPE(mp), ==, M_DATA);

	ASSERT(stmp->stm_mp == NULL);
	stmp->stm_mp = mp;

	stmp->stm_base = (caddr_t)(mp->b_rptr);
	stmp->stm_size = MBLKL(mp);

	/* Bind the STREAMS block to the mapping */
	rc = ddi_dma_addr_bind_handle(stmp->stm_dma_handle, NULL,
	    stmp->stm_base, stmp->stm_size, DDI_DMA_WRITE | DDI_DMA_STREAMING,
	    DDI_DMA_DONTWAIT, NULL, &dmac, &ncookies);
	if (rc != DDI_DMA_MAPPED)
		goto fail1;

	ASSERT3U(ncookies, <=, SFXGE_TX_MAPPING_NADDR);

	/*
	 * Construct an array of addresses and an initial
	 * offset.
	 */
	n = 0;
	stmp->stm_addr[n++] = dmac.dmac_laddress & SFXGE_TX_DESCMASK;
	DTRACE_PROBE1(addr, uint64_t, dmac.dmac_laddress & SFXGE_TX_DESCMASK);

	stmp->stm_off = dmac.dmac_laddress & SFXGE_TX_DESCOFFSET;

	size = MIN(SFXGE_TX_DESCSIZE - stmp->stm_off, dmac.dmac_size);
	dmac.dmac_laddress += size;
	dmac.dmac_size -= size;

	for (;;) {
		ASSERT3U(n, <, SFXGE_TX_MAPPING_NADDR);

		if (dmac.dmac_size == 0) {
			if (--ncookies == 0)
				break;

			ddi_dma_nextcookie(stmp->stm_dma_handle, &dmac);
		}

		ASSERT((dmac.dmac_laddress & SFXGE_TX_DESCMASK) != 0);
		ASSERT((dmac.dmac_laddress & SFXGE_TX_DESCOFFSET) == 0);
		stmp->stm_addr[n++] = dmac.dmac_laddress;
		DTRACE_PROBE1(addr, uint64_t, dmac.dmac_laddress);

		size = MIN(SFXGE_TX_DESCSIZE, dmac.dmac_size);
		dmac.dmac_laddress += size;
		dmac.dmac_size -= size;
	}
	ASSERT3U(n, <=, SFXGE_TX_MAPPING_NADDR);

	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	stmp->stm_size = 0;
	stmp->stm_base = NULL;

	stmp->stm_mp = NULL;

	return (-1);
}

static void
sfxge_tx_qreap(sfxge_txq_t *stp)
{
	unsigned int reaped;

	ASSERT(mutex_owned(&(stp->st_lock)));

	reaped = stp->st_reaped;
	while (reaped != stp->st_completed) {
		unsigned int id;
		sfxge_tx_mapping_t *stmp;
		sfxge_tx_buffer_t *stbp;

		id = reaped++ & (SFXGE_TX_NDESCS - 1);

		ASSERT3P(stp->st_mp[id], ==, NULL);

		if ((stmp = stp->st_stmp[id]) != NULL) {
			stp->st_stmp[id] = NULL;

			/* Free all the mappings */
			do {
				sfxge_tx_mapping_t *next;

				next = stmp->stm_next;
				stmp->stm_next = NULL;

				sfxge_tx_qfmp_put(stp, stmp);

				stmp = next;
			} while (stmp != NULL);
		}

		if ((stbp = stp->st_stbp[id]) != NULL) {
			stp->st_stbp[id] = NULL;

			/* Free all the buffers */
			do {
				sfxge_tx_buffer_t *next;

				next = stbp->stb_next;
				stbp->stb_next = NULL;

				stbp->stb_esm.esm_used = 0;
				stbp->stb_off = 0;

				sfxge_tx_qfbp_put(stp, stbp);

				stbp = next;
			} while (stbp != NULL);
		}
	}
	stp->st_reaped = reaped;
}

static void
sfxge_tx_qlist_abort(sfxge_txq_t *stp)
{
	unsigned int id;
	sfxge_tx_mapping_t *stmp;
	sfxge_tx_buffer_t *stbp;
	mblk_t *mp;

	ASSERT(mutex_owned(&(stp->st_lock)));

	id = stp->st_added & (SFXGE_TX_NDESCS - 1);

	/* Clear the completion information */
	stmp = stp->st_stmp[id];
	stp->st_stmp[id] = NULL;

	/* Free any mappings that were used */
	while (stmp != NULL) {
		sfxge_tx_mapping_t *next;

		next = stmp->stm_next;
		stmp->stm_next = NULL;

		if (stmp->stm_mp != NULL)
			sfxge_tx_msgb_unbind(stmp);

		sfxge_tx_qfmp_put(stp, stmp);

		stmp = next;
	}

	stbp = stp->st_stbp[id];
	stp->st_stbp[id] = NULL;

	/* Free any buffers that were used */
	while (stbp != NULL) {
		sfxge_tx_buffer_t *next;

		next = stbp->stb_next;
		stbp->stb_next = NULL;

		stbp->stb_off = 0;
		stbp->stb_esm.esm_used = 0;

		sfxge_tx_qfbp_put(stp, stbp);

		stbp = next;
	}

	mp = stp->st_mp[id];
	stp->st_mp[id] = NULL;

	if (mp != NULL)
		freemsg(mp);

	/* Clear the fragment list */
	stp->st_n = 0;
}

/* Push descriptors to the TX ring setting blocked if no space */
static void
sfxge_tx_qlist_post(sfxge_txq_t *stp)
{
	unsigned int id;
	unsigned int level;
	unsigned int available;
	int rc;

	ASSERT(mutex_owned(&(stp->st_lock)));

	ASSERT(stp->st_n != 0);

again:
	level = stp->st_added - stp->st_reaped;
	available = EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) - level;

	id = stp->st_added & (SFXGE_TX_NDESCS - 1);

	if (available < stp->st_n) {
		rc = ENOSPC;
		goto fail1;
	}

	ASSERT3U(available, >=, stp->st_n);

	/* Post the fragment list */
	if ((rc = efx_tx_qpost(stp->st_etp, stp->st_eb, stp->st_n,
	    stp->st_reaped, &(stp->st_added))) != 0)
		goto fail2;

	/*
	 * If the list took more than a single descriptor then we need to
	 * to move the completion information so it is referenced by the last
	 * descriptor.
	 */
	if (((stp->st_added - 1) & (SFXGE_TX_NDESCS - 1)) != id) {
		sfxge_tx_mapping_t *stmp;
		sfxge_tx_buffer_t *stbp;
		mblk_t *mp;

		stmp = stp->st_stmp[id];
		stp->st_stmp[id] = NULL;

		stbp = stp->st_stbp[id];
		stp->st_stbp[id] = NULL;

		mp = stp->st_mp[id];
		stp->st_mp[id] = NULL;

		id = (stp->st_added - 1) & (SFXGE_TX_NDESCS - 1);

		ASSERT(stp->st_stmp[id] == NULL);
		stp->st_stmp[id] = stmp;

		ASSERT(stp->st_stbp[id] == NULL);
		stp->st_stbp[id] = stbp;

		ASSERT(stp->st_mp[id] == NULL);
		stp->st_mp[id] = mp;
	}

	/* Clear the list */
	stp->st_n = 0;

	ASSERT3U(stp->st_unblock, ==, SFXGE_TXQ_NOT_BLOCKED);
	return;

fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	ASSERT(rc == ENOSPC);

	level = stp->st_added - stp->st_completed;
	available = EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) - level;

	/*
	 * If there would be enough space after we've reaped any completed
	 * mappings and buffers, and we gain sufficient queue space by doing
	 * so, then reap now and try posting again.
	 */
	if (stp->st_n <= available &&
	    stp->st_completed - stp->st_reaped >= SFXGE_TX_BATCH) {
		sfxge_tx_qreap(stp);

		goto again;
	}

	/* Set the unblock level */
	if (stp->st_unblock == SFXGE_TXQ_NOT_BLOCKED) {
		stp->st_unblock = SFXGE_TXQ_UNBLOCK_LEVEL1;
	} else {
		ASSERT(stp->st_unblock == SFXGE_TXQ_UNBLOCK_LEVEL1);

		stp->st_unblock = SFXGE_TXQ_UNBLOCK_LEVEL2;
	}

	/*
	 * Avoid a race with completion interrupt handling that could leave the
	 * queue blocked.
	 *
	 * NOTE: The use of st_pending rather than st_completed is intentional
	 *	 as st_pending is updated per-event rather than per-batch and
	 *	 therefore avoids needless deferring.
	 */
	if (stp->st_pending == stp->st_added) {
		sfxge_tx_qreap(stp);

		stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;
		goto again;
	}

	ASSERT(stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED);
}

static int
sfxge_tx_kstat_update(kstat_t *ksp, int rw)
{
	sfxge_txq_t *stp = ksp->ks_private;
	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
	kstat_named_t *knp;
	int rc;

	ASSERT(mutex_owned(&(stp->st_lock)));

	if (rw != KSTAT_READ) {
		rc = EACCES;
		goto fail1;
	}

	if (stp->st_state != SFXGE_TXQ_STARTED)
		goto done;

	efx_tx_qstats_update(stp->st_etp, stp->st_stat);
	knp = (kstat_named_t *)ksp->ks_data + TX_NQSTATS;
	knp->value.ui64 = stdp->get_pkt_limit;
	knp++;
	knp->value.ui64 = stdp->put_pkt_limit;
	knp++;
	knp->value.ui64 = stdp->get_full_count;
	knp++;
	knp->value.ui64 = stdp->put_full_count;

done:
	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static int
sfxge_tx_kstat_init(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	unsigned int index = stp->st_index;
	dev_info_t *dip = sp->s_dip;
	kstat_t *ksp;
	kstat_named_t *knp;
	char name[MAXNAMELEN];
	unsigned int id;
	int rc;

	/* Create the set */
	(void) snprintf(name, MAXNAMELEN - 1, "%s_txq%04d",
	    ddi_driver_name(dip), index);

	if ((ksp = kstat_create((char *)ddi_driver_name(dip),
	    ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED,
	    TX_NQSTATS + 4, 0)) == NULL) {
		rc = ENOMEM;
		goto fail1;
	}

	stp->st_ksp = ksp;

	ksp->ks_update = sfxge_tx_kstat_update;
	ksp->ks_private = stp;
	ksp->ks_lock = &(stp->st_lock);

	/* Initialise the named stats */
	stp->st_stat = knp = ksp->ks_data;
	for (id = 0; id < TX_NQSTATS; id++) {
		kstat_named_init(knp, (char *)efx_tx_qstat_name(sp->s_enp, id),
		    KSTAT_DATA_UINT64);
		knp++;
	}
	kstat_named_init(knp, "dpl_get_pkt_limit", KSTAT_DATA_UINT64);
	knp++;
	kstat_named_init(knp, "dpl_put_pkt_limit", KSTAT_DATA_UINT64);
	knp++;
	kstat_named_init(knp, "dpl_get_full_count", KSTAT_DATA_UINT64);
	knp++;
	kstat_named_init(knp, "dpl_put_full_count", KSTAT_DATA_UINT64);

	kstat_install(ksp);
	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static void
sfxge_tx_kstat_fini(sfxge_txq_t *stp)
{
	/* Destroy the set */
	kstat_delete(stp->st_ksp);
	stp->st_ksp = NULL;
	stp->st_stat = NULL;
}

static int
sfxge_tx_qinit(sfxge_t *sp, unsigned int index, sfxge_txq_type_t type,
    unsigned int evq)
{
	sfxge_txq_t *stp;
	sfxge_tx_dpl_t *stdp;
	int rc;

	ASSERT3U(index, <, EFX_ARRAY_SIZE(sp->s_stp));
	ASSERT3U(type, <, SFXGE_TXQ_NTYPES);
	ASSERT3U(evq, <, EFX_ARRAY_SIZE(sp->s_sep));

	if ((stp = kmem_cache_alloc(sp->s_tqc, KM_SLEEP)) == NULL) {
		rc = ENOMEM;
		goto fail1;
	}
	ASSERT3U(stp->st_state, ==, SFXGE_TXQ_UNINITIALIZED);

	stdp = &(stp->st_dpl);

	stp->st_index = index;
	stp->st_type = type;
	stp->st_evq = evq;

	mutex_init(&(stp->st_lock), NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(sp->s_intr.si_intr_pri));

	/* Initialize the statistics */
	if ((rc = sfxge_tx_kstat_init(stp)) != 0)
		goto fail2;

	stdp->get_pkt_limit = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
	    DDI_PROP_DONTPASS, "tx_dpl_get_pkt_limit",
	    SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT);

	stdp->put_pkt_limit = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
	    DDI_PROP_DONTPASS, "tx_dpl_put_pkt_limit",
	    SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT);

	/* Allocate a per-EVQ label for events from this TXQ */
	if ((rc = sfxge_ev_txlabel_alloc(sp, evq, stp, &(stp->st_label))) != 0)
		goto fail2;

	stp->st_state = SFXGE_TXQ_INITIALIZED;

	/* Attach the TXQ to the driver */
	ASSERT3P(sp->s_stp[index], ==, NULL);
	sp->s_stp[index] = stp;
	sp->s_tx_qcount++;

	return (0);

fail2:
	DTRACE_PROBE(fail2);

	sfxge_tx_kstat_fini(stp);


	stp->st_evq = 0;
	stp->st_type = 0;
	stp->st_index = 0;

	mutex_destroy(&(stp->st_lock));

	kmem_cache_free(sp->s_tqc, stp);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static int
sfxge_tx_qstart(sfxge_t *sp, unsigned int index)
{
	sfxge_txq_t *stp = sp->s_stp[index];
	efx_nic_t *enp = sp->s_enp;
	efsys_mem_t *esmp;
	sfxge_evq_t *sep;
	unsigned int evq;
	unsigned int flags;
	unsigned int desc_index;
	int rc;

	mutex_enter(&(stp->st_lock));

	esmp = &(stp->st_mem);
	evq = stp->st_evq;
	sep = sp->s_sep[evq];

	ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED);
	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);

	/* Zero the memory */
	bzero(esmp->esm_base, EFX_TXQ_SIZE(SFXGE_TX_NDESCS));

	/* Program the buffer table */
	if ((rc = sfxge_sram_buf_tbl_set(sp, stp->st_id, esmp,
	    EFX_TXQ_NBUFS(SFXGE_TX_NDESCS))) != 0)
		goto fail1;

	switch (stp->st_type) {
	case SFXGE_TXQ_NON_CKSUM:
		flags = 0;
		break;

	case SFXGE_TXQ_IP_CKSUM:
		flags = EFX_TXQ_CKSUM_IPV4;
		break;

	case SFXGE_TXQ_IP_TCP_UDP_CKSUM:
		flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP;
		break;

	default:
		ASSERT(B_FALSE);

		flags = 0;
		break;
	}

	/* Create the transmit queue */
	if ((rc = efx_tx_qcreate(enp, index, stp->st_label, esmp,
	    SFXGE_TX_NDESCS, stp->st_id, flags, sep->se_eep,
	    &(stp->st_etp), &desc_index)) != 0)
		goto fail2;

	/* Initialise queue descriptor indexes */
	stp->st_added = desc_index;
	stp->st_pending = desc_index;
	stp->st_completed = desc_index;
	stp->st_reaped = desc_index;

	/* Enable the transmit queue */
	efx_tx_qenable(stp->st_etp);

	stp->st_state = SFXGE_TXQ_STARTED;

	mutex_exit(&(stp->st_lock));

	return (0);

fail2:
	DTRACE_PROBE(fail2);

	/* Clear entries from the buffer table */
	sfxge_sram_buf_tbl_clear(sp, stp->st_id,
	    EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	mutex_exit(&(stp->st_lock));

	return (rc);
}

static inline int
sfxge_tx_qmapping_add(sfxge_txq_t *stp, sfxge_tx_mapping_t *stmp,
    size_t *offp, size_t *limitp)
{
	mblk_t *mp;
	size_t mapping_off;
	size_t mapping_size;
	int rc;

	ASSERT3U(*offp, <, stmp->stm_size);
	ASSERT(*limitp != 0);

	mp = stmp->stm_mp;

	ASSERT3P(stmp->stm_base, ==, mp->b_rptr);
	ASSERT3U(stmp->stm_size, ==, MBLKL(mp));

	mapping_off = stmp->stm_off + *offp;
	mapping_size = stmp->stm_size - *offp;

	while (mapping_size != 0 && *limitp != 0) {
		size_t page =
		    mapping_off >> SFXGE_TX_DESCSHIFT;
		size_t page_off =
		    mapping_off & SFXGE_TX_DESCOFFSET;
		size_t page_size =
		    SFXGE_TX_DESCSIZE - page_off;
		efx_buffer_t *ebp;

		ASSERT3U(page, <, SFXGE_TX_MAPPING_NADDR);
		ASSERT((stmp->stm_addr[page] & SFXGE_TX_DESCMASK) != 0);

		page_size = MIN(page_size, mapping_size);
		page_size = MIN(page_size, *limitp);

		ASSERT3U(stp->st_n, <=,
		    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
		if (stp->st_n ==
		    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS)) {
			rc = ENOSPC;
			goto fail1;
		}

		ebp = &(stp->st_eb[stp->st_n++]);
		ebp->eb_addr = stmp->stm_addr[page] +
		    page_off;
		ebp->eb_size = page_size;

		*offp += page_size;
		*limitp -= page_size;

		mapping_off += page_size;
		mapping_size -= page_size;

		ebp->eb_eop = (*limitp == 0 ||
		    (mapping_size == 0 && mp->b_cont == NULL));

		DTRACE_PROBE5(tx_mapping_add,
		    unsigned int, stp->st_index,
		    unsigned int, stp->st_n - 1,
		    uint64_t, ebp->eb_addr,
		    size_t, ebp->eb_size,
		    boolean_t, ebp->eb_eop);
	}

	ASSERT3U(*offp, <=, stmp->stm_size);

	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static inline int
sfxge_tx_qbuffer_add(sfxge_txq_t *stp, sfxge_tx_buffer_t *stbp, boolean_t eop)
{
	efx_buffer_t *ebp;
	int rc;

	ASSERT3U(stp->st_n, <=,
	    EFX_TXQ_LIMIT(SFXGE_TX_NDESCS));
	if (stp->st_n == EFX_TXQ_LIMIT(SFXGE_TX_NDESCS)) {
		rc = ENOSPC;
		goto fail1;
	}

	ebp = &(stp->st_eb[stp->st_n++]);
	ebp->eb_addr = stbp->stb_esm.esm_addr + stbp->stb_off;
	ebp->eb_size = stbp->stb_esm.esm_used - stbp->stb_off;
	ebp->eb_eop = eop;

	(void) ddi_dma_sync(stbp->stb_esm.esm_dma_handle,
	    stbp->stb_off, ebp->eb_size,
	    DDI_DMA_SYNC_FORDEV);

	stbp->stb_off = stbp->stb_esm.esm_used;

	DTRACE_PROBE5(tx_buffer_add,
	    unsigned int, stp->st_index,
	    unsigned int, stp->st_n - 1,
	    uint64_t, ebp->eb_addr, size_t, ebp->eb_size,
	    boolean_t, ebp->eb_eop);

	return (0);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static inline boolean_t
sfxge_tx_msgb_copy(mblk_t *mp, sfxge_tx_buffer_t *stbp, size_t *offp,
    size_t *limitp)
{
	size_t data_off;
	size_t data_size;
	size_t copy_off;
	size_t copy_size;
	boolean_t eop;

	ASSERT3U(*offp, <=, MBLKL(mp));
	ASSERT(*limitp != 0);

	data_off = *offp;
	data_size = MBLKL(mp) - *offp;

	copy_off = stbp->stb_esm.esm_used;
	copy_size = SFXGE_TX_BUFFER_SIZE - copy_off;

	copy_size = MIN(copy_size, data_size);
	copy_size = MIN(copy_size, *limitp);

	bcopy(mp->b_rptr + data_off,
	    stbp->stb_esm.esm_base + copy_off, copy_size);

	stbp->stb_esm.esm_used += copy_size;
	ASSERT3U(stbp->stb_esm.esm_used, <=,
	    SFXGE_TX_BUFFER_SIZE);

	*offp += copy_size;
	*limitp -= copy_size;

	data_off += copy_size;
	data_size -= copy_size;

	eop = (*limitp == 0 ||
	    (data_size == 0 && mp->b_cont == NULL));

	ASSERT3U(*offp, <=, MBLKL(mp));

	return (eop);
}

static int
sfxge_tx_qpayload_fragment(sfxge_txq_t *stp, unsigned int id, mblk_t **mpp,
    size_t *offp, size_t size, boolean_t copy)
{
	sfxge_t *sp = stp->st_sp;
	mblk_t *mp = *mpp;
	size_t off = *offp;
	sfxge_tx_buffer_t *stbp;
	sfxge_tx_mapping_t *stmp;
	int rc;

	stbp = stp->st_stbp[id];
	ASSERT(stbp == NULL || (stbp->stb_esm.esm_used == stbp->stb_off));

	stmp = stp->st_stmp[id];

	while (size != 0) {
		boolean_t eop;

		ASSERT(mp != NULL);

		if (mp->b_cont != NULL)
			prefetch_read_many(mp->b_cont);

		ASSERT3U(off, <, MBLKL(mp));

		if (copy)
			goto copy;

		/*
		 * Check whether we have already mapped this data block for
		 * DMA.
		 */
		if (stmp == NULL || stmp->stm_mp != mp) {
			/*
			 * If we are part way through copying a data block then
			 * there's no point in trying to map it for DMA.
			 */
			if (off != 0)
				goto copy;

			/*
			 * If the data block is too short then the cost of
			 * mapping it for DMA would outweigh the cost of
			 * copying it.
			 */
			if (MBLKL(mp) < SFXGE_TX_COPY_THRESHOLD)
				goto copy;

			/* Try to grab a transmit mapping from the pool */
			stmp = sfxge_tx_qfmp_get(stp);
			if (stmp == NULL) {
				/*
				 * The pool was empty so allocate a new
				 * mapping.
				 */
				if ((stmp = kmem_cache_alloc(sp->s_tmc,
				    KM_NOSLEEP)) == NULL)
					goto copy;
			}

			/* Add the DMA mapping to the list */
			stmp->stm_next = stp->st_stmp[id];
			stp->st_stmp[id] = stmp;

			/* Try to bind the data block to the mapping */
			if (sfxge_tx_msgb_bind(mp, stmp) != 0)
				goto copy;
		}
		ASSERT3P(stmp->stm_mp, ==, mp);

		/*
		 * If we have a partially filled buffer then we must add it to
		 * the fragment list before adding the mapping.
		 */
		if (stbp != NULL && (stbp->stb_esm.esm_used > stbp->stb_off)) {
			rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE);
			if (rc != 0)
				goto fail1;
		}

		/* Add the mapping to the fragment list */
		rc = sfxge_tx_qmapping_add(stp, stmp, &off, &size);
		if (rc != 0)
			goto fail2;

		ASSERT(off == MBLKL(mp) || size == 0);

		/*
		 * If the data block has been exhausted then Skip over the
		 * control block and advance to the next data block.
		 */
		if (off == MBLKL(mp)) {
			mp = mp->b_cont;
			off = 0;
		}

		continue;

copy:
		if (stbp == NULL ||
		    stbp->stb_esm.esm_used == SFXGE_TX_BUFFER_SIZE) {
			/* Try to grab a buffer from the pool */
			stbp = sfxge_tx_qfbp_get(stp);
			if (stbp == NULL) {
				/*
				 * The pool was empty so allocate a new
				 * buffer.
				 */
				if ((stbp = kmem_cache_alloc(sp->s_tbc,
				    KM_NOSLEEP)) == NULL) {
					rc = ENOMEM;
					goto fail3;
				}
			}

			/* Add it to the list */
			stbp->stb_next = stp->st_stbp[id];
			stp->st_stbp[id] = stbp;
		}

		/* Copy as much of the data block as we can into the buffer */
		eop = sfxge_tx_msgb_copy(mp, stbp, &off, &size);

		ASSERT(off == MBLKL(mp) || size == 0 ||
		    stbp->stb_esm.esm_used == SFXGE_TX_BUFFER_SIZE);

		/*
		 * If we have reached the end of the packet, or the buffer is
		 * full, then add the buffer to the fragment list.
		 */
		if (stbp->stb_esm.esm_used == SFXGE_TX_BUFFER_SIZE || eop) {
			rc = sfxge_tx_qbuffer_add(stp, stbp, eop);
			if (rc != 0)
				goto fail4;
		}

		/*
		 * If the data block has been exhaused then advance to the next
		 * one.
		 */
		if (off == MBLKL(mp)) {
			mp = mp->b_cont;
			off = 0;
		}
	}

	*mpp = mp;
	*offp = off;

	return (0);

fail4:
	DTRACE_PROBE(fail4);
fail3:
	DTRACE_PROBE(fail3);
fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static int
sfxge_tx_qlso_fragment(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp,
    boolean_t copy)
{
	sfxge_t *sp = stp->st_sp;
	mblk_t *mp = stpp->stp_mp;
	struct ether_header *etherhp = stpp->stp_etherhp;
	struct ip *iphp = stpp->stp_iphp;
	struct tcphdr *thp = stpp->stp_thp;
	size_t size = stpp->stp_size;
	size_t off = stpp->stp_off;
	size_t mss = stpp->stp_mss;
	unsigned int id;
	caddr_t hp;
	size_t ehs, hs;
	uint16_t start_len;
	uint16_t start_id;
	uint16_t ip_id;
	uint8_t start_flags;
	uint32_t start_seq;
	uint32_t th_seq;
	size_t lss;
	sfxge_tx_buffer_t *stbp;
	int rc;

	ASSERT(mutex_owned(&(stp->st_lock)));

	if ((DB_LSOFLAGS(mp) & HW_LSO) == 0) {
		rc = EINVAL;
		goto fail1;
	}

	id = stp->st_added & (SFXGE_TX_NDESCS - 1);

	ASSERT(stp->st_n == 0);
	ASSERT(stp->st_stbp[id] == NULL);
	ASSERT(stp->st_stmp[id] == NULL);

	ehs = (etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
	    sizeof (struct ether_vlan_header) :
	    sizeof (struct ether_header);
	if (msgdsize(mp) != ehs + ntohs(iphp->ip_len)) {
		rc = EINVAL;
		goto fail2;
	}

	/* The payload offset is equivalent to the size of the headers */
	hp = (caddr_t)(mp->b_rptr);
	hs = off;

	/*
	 * If the initial data block only contains the headers then advance
	 * to the next one.
	 */
	if (hs > MBLKL(mp)) {
		rc = EINVAL;
		goto fail3;
	}
	mp->b_rptr += hs;

	if (MBLKL(mp) == 0)
		mp = mp->b_cont;

	off = 0;

	/* Check IP and TCP headers are suitable for LSO */
	if (((iphp->ip_off & ~htons(IP_DF)) != 0) ||
	    ((thp->th_flags & (TH_URG | TH_SYN)) != 0) ||
	    (thp->th_urp != 0)) {
		rc = EINVAL;
		goto fail4;
	}

	if (size + (thp->th_off << 2) + (iphp->ip_hl << 2) !=
	    ntohs(iphp->ip_len)) {
		rc = EINVAL;
		goto fail4;
	}

	/*
	 * Get the base IP id, The stack leaves enough of a gap in id space
	 * for us to increment this for each segment we send out.
	 */
	start_len = ntohs(iphp->ip_len);
	start_id = ip_id = ntohs(iphp->ip_id);

	/* Get the base TCP sequence number and flags */
	start_flags = thp->th_flags;
	start_seq = th_seq = ntohl(thp->th_seq);

	/* Adjust the header for interim segments */
	iphp->ip_len = htons((iphp->ip_hl << 2) + (thp->th_off << 2) + mss);
	thp->th_flags = start_flags & ~(TH_PUSH | TH_FIN);

	lss = size;
	if ((lss / mss) >= (EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 2)) {
		rc = EINVAL;
		goto fail5;
	}

	stbp = NULL;
	while (lss != 0) {
		size_t ss = MIN(lss, mss);
		boolean_t eol = (ss == lss);

		/* Adjust the header for this segment */
		iphp->ip_id = htons(ip_id);
		ip_id++;

		thp->th_seq = htonl(th_seq);
		th_seq += ss;

		/* If this is the final segment then do some extra adjustment */
		if (eol) {
			iphp->ip_len = htons((iphp->ip_hl << 2) +
			    (thp->th_off << 2) + ss);
			thp->th_flags = start_flags;
		}

		if (stbp == NULL ||
		    stbp->stb_esm.esm_used + hs > SFXGE_TX_BUFFER_SIZE) {
			/* Try to grab a buffer from the pool */
			stbp = sfxge_tx_qfbp_get(stp);
			if (stbp == NULL) {
				/*
				 * The pool was empty so allocate a new
				 * buffer.
				 */
				if ((stbp = kmem_cache_alloc(sp->s_tbc,
				    KM_NOSLEEP)) == NULL) {
					rc = ENOMEM;
					goto fail6;
				}
			}

			/* Add it to the list */
			stbp->stb_next = stp->st_stbp[id];
			stp->st_stbp[id] = stbp;
		}

		/* Copy in the headers */
		ASSERT3U(stbp->stb_off, ==, stbp->stb_esm.esm_used);
		bcopy(hp, stbp->stb_esm.esm_base + stbp->stb_off, hs);
		stbp->stb_esm.esm_used += hs;

		/* Add the buffer to the fragment list */
		rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE);
		if (rc != 0)
			goto fail7;

		/* Add the payload to the fragment list */
		if ((rc = sfxge_tx_qpayload_fragment(stp, id, &mp, &off,
		    ss, copy)) != 0)
			goto fail8;

		lss -= ss;
	}
	ASSERT3U(off, ==, 0);
	ASSERT3P(mp, ==, NULL);

	ASSERT3U(th_seq - start_seq, ==, size);

	/*
	 * If no part of the packet has been mapped for DMA then we can free
	 * it now, otherwise it can only be freed on completion.
	 */
	if (stp->st_stmp[id] == NULL)
		freemsg(stpp->stp_mp);
	else
		stp->st_mp[id] = stpp->stp_mp;

	stpp->stp_mp = NULL;

	return (0);

fail8:
	DTRACE_PROBE(fail8);
fail7:
	DTRACE_PROBE(fail7);
fail6:
	DTRACE_PROBE(fail6);
fail5:
	DTRACE_PROBE(fail5);

	/* Restore the header */
	thp->th_seq = htonl(start_seq);
	thp->th_flags = start_flags;

	iphp->ip_len = htons(start_len);
	iphp->ip_id = htons(start_id);

fail4:
	DTRACE_PROBE(fail4);

	mp = stpp->stp_mp;
	mp->b_rptr -= hs;

	ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
	    sizeof (struct ether_vlan_header) :
	    sizeof (struct ether_header)) +
	    ntohs(iphp->ip_len), ==, msgdsize(mp));

	ASSERT(stp->st_mp[id] == NULL);

fail3:
	DTRACE_PROBE(fail3);
fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}

static int
sfxge_tx_qpacket_fragment(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp,
    boolean_t copy)
{
	sfxge_t *sp = stp->st_sp;
	mblk_t *mp = stpp->stp_mp;
	unsigned int id;
	size_t off;
	size_t size;
	sfxge_tx_mapping_t *stmp;
	sfxge_tx_buffer_t *stbp;
	int rc;

	ASSERT(mutex_owned(&(stp->st_lock)));

	ASSERT(stp->st_n == 0);

	id = stp->st_added & (SFXGE_TX_NDESCS - 1);

	ASSERT(stp->st_stbp[id] == NULL);
	ASSERT(stp->st_stmp[id] == NULL);

	off = 0;
	size = LONG_MAX;	/* must be larger than the packet */

	stbp = NULL;
	stmp = NULL;

	while (mp != NULL) {
		boolean_t eop;

		ASSERT(mp != NULL);

		if (mp->b_cont != NULL)
			prefetch_read_many(mp->b_cont);

		ASSERT(stmp == NULL || stmp->stm_mp != mp);

		if (copy)
			goto copy;

		/*
		 * If we are part way through copying a data block then there's
		 * no point in trying to map it for DMA.
		 */
		if (off != 0)
			goto copy;

		/*
		 * If the data block is too short then the cost of mapping it
		 * for DMA would outweigh the cost of copying it.
		 *
		 * TX copy break
		 */
		if (MBLKL(mp) < SFXGE_TX_COPY_THRESHOLD)
			goto copy;

		/* Try to grab a transmit mapping from the pool */
		stmp = sfxge_tx_qfmp_get(stp);
		if (stmp == NULL) {
			/*
			 * The pool was empty so allocate a new
			 * mapping.
			 */
			if ((stmp = kmem_cache_alloc(sp->s_tmc,
			    KM_NOSLEEP)) == NULL)
				goto copy;
		}

		/* Add the DMA mapping to the list */
		stmp->stm_next = stp->st_stmp[id];
		stp->st_stmp[id] = stmp;

		/* Try to bind the data block to the mapping */
		if (sfxge_tx_msgb_bind(mp, stmp) != 0)
			goto copy;

		/*
		 * If we have a partially filled buffer then we must add it to
		 * the fragment list before adding the mapping.
		 */
		if (stbp != NULL && (stbp->stb_esm.esm_used > stbp->stb_off)) {
			rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE);
			if (rc != 0)
				goto fail1;
		}

		/* Add the mapping to the fragment list */
		rc = sfxge_tx_qmapping_add(stp, stmp, &off, &size);
		if (rc != 0)
			goto fail2;

		ASSERT3U(off, ==, MBLKL(mp));

		/* Advance to the next data block */
		mp = mp->b_cont;
		off = 0;
		continue;

copy:
		if (stbp == NULL ||
		    stbp->stb_esm.esm_used == SFXGE_TX_BUFFER_SIZE) {
			/* Try to grab a buffer from the pool */
			stbp = sfxge_tx_qfbp_get(stp);
			if (stbp == NULL) {
				/*
				 * The pool was empty so allocate a new
				 * buffer.
				 */
				if ((stbp = kmem_cache_alloc(sp->s_tbc,
				    KM_NOSLEEP)) == NULL) {
					rc = ENOMEM;
					goto fail3;
				}
			}

			/* Add it to the list */
			stbp->stb_next = stp->st_stbp[id];
			stp->st_stbp[id] = stbp;
		}

		/* Copy as much of the data block as we can into the buffer */
		eop = sfxge_tx_msgb_copy(mp, stbp, &off, &size);

		ASSERT(off == MBLKL(mp) ||
		    stbp->stb_esm.esm_used == SFXGE_TX_BUFFER_SIZE);

		/*
		 * If we have reached the end of the packet, or the buffer is
		 * full, then add the buffer to the fragment list.
		 */
		if (stbp->stb_esm.esm_used == SFXGE_TX_BUFFER_SIZE || eop) {
			rc = sfxge_tx_qbuffer_add(stp, stbp, eop);
			if (rc != 0)
				goto fail4;
		}

		/*
		 * If the data block has been exhaused then advance to the next
		 * one.
		 */
		if (off == MBLKL(mp)) {
			mp = mp->b_cont;
			off = 0;
		}
	}
	ASSERT3U(off, ==, 0);
	ASSERT3P(mp, ==, NULL);
	ASSERT3U(size, !=, 0);

	/*
	 * If no part of the packet has been mapped for DMA then we can free
	 * it now, otherwise it can only be freed on completion.
	 */
	if (stp->st_stmp[id] == NULL)
		freemsg(stpp->stp_mp);
	else
		stp->st_mp[id] = stpp->stp_mp;

	stpp->stp_mp = NULL;

	return (0);

fail4:
	DTRACE_PROBE(fail4);
fail3:
	DTRACE_PROBE(fail3);
fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	ASSERT(stp->st_stmp[id] == NULL);

	return (rc);
}


#define	SFXGE_TX_QDPL_PUT_PENDING(_stp)					\
	((_stp)->st_dpl.std_put != 0)

static void
sfxge_tx_qdpl_swizzle(sfxge_txq_t *stp)
{
	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
	volatile uintptr_t *putp;
	uintptr_t put;
	sfxge_tx_packet_t *stpp;
	sfxge_tx_packet_t *p;
	sfxge_tx_packet_t **pp;
	unsigned int count;

	ASSERT(mutex_owned(&(stp->st_lock)));

	/*
	 * Guaranteed that in flight TX packets will cause more TX completions
	 * hence more swizzles must happen
	 */
	ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));
	if (stdp->std_count >= stdp->get_pkt_limit)
		return;

	/* Acquire the put list - replacing with an empty list */
	putp = &(stdp->std_put);
	put = atomic_swap_ulong(putp, 0);
	stpp = (void *)put;

	if (stpp == NULL)
		return;

	/* Reverse the list */
	pp = &(stpp->stp_next);
	p = NULL;

	count = 0;
	do {
		sfxge_tx_packet_t *next;

		next = stpp->stp_next;

		stpp->stp_next = p;
		p = stpp;

		count++;
		stpp = next;
	} while (stpp != NULL);

	/* Add it to the tail of the get list */
	ASSERT3P(*pp, ==, NULL);

	*(stdp->std_getp) = p;
	stdp->std_getp = pp;
	stdp->std_count += count;
	ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));

	DTRACE_PROBE2(dpl_counts, int, stdp->std_count, int, count);
}


/*
 * If TXQ locked, add the RX DPL put list and this packet to the TX DPL get list
 * If TXQ unlocked, atomically add this packet to TX DPL put list
 *
 * The only possible error is ENOSPC (used for TX backpressure)
 * For the TX DPL put or get list becoming full, in both cases there must be
 * future TX completions (as represented by the packets on the DPL get lists).
 *
 * This ensures that in the future mac_tx_update() will be called from
 * sfxge_tx_qcomplete()
 */
static inline int
sfxge_tx_qdpl_add(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp, int locked)
{
	sfxge_tx_dpl_t *stdp = &stp->st_dpl;

	ASSERT3P(stpp->stp_next, ==, NULL);

	if (locked) {
		ASSERT(mutex_owned(&stp->st_lock));

		if (stdp->std_count >= stdp->get_pkt_limit) {
			stdp->get_full_count++;
			return (ENOSPC);
		}

		/* Reverse the put list onto the get list */
		sfxge_tx_qdpl_swizzle(stp);

		/* Add to the tail of the get list */
		*(stdp->std_getp) = stpp;
		stdp->std_getp = &stpp->stp_next;
		stdp->std_count++;
		ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));

	} else {
		volatile uintptr_t *putp;
		uintptr_t old;
		uintptr_t new;
		sfxge_tx_packet_t *old_pkt;

		putp = &(stdp->std_put);
		new = (uintptr_t)stpp;

		/* Add to the head of the put list, keeping a list length */
		do {
			old = *putp;
			old_pkt =  (sfxge_tx_packet_t *)old;

			stpp->stp_dpl_put_len = old ?
			    old_pkt->stp_dpl_put_len + 1 : 1;

			if (stpp->stp_dpl_put_len >= stdp->put_pkt_limit) {
				stpp->stp_next = 0;
				stpp->stp_dpl_put_len = 0;
				stdp->put_full_count++;
				return (ENOSPC);
			}

			stpp->stp_next = (void *)old;
		} while (atomic_cas_ulong(putp, old, new) != old);
	}
	return (0);
}


/* Take all packets from DPL get list and try to send to HW */
static void
sfxge_tx_qdpl_drain(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
	unsigned int pushed = stp->st_added;
	sfxge_tx_packet_t *stpp;
	unsigned int count;

	ASSERT(mutex_owned(&(stp->st_lock)));

	prefetch_read_many(sp->s_enp);
	prefetch_read_many(stp->st_etp);

	stpp = stdp->std_get;
	count = stdp->std_count;

	while (count != 0) {
		sfxge_tx_packet_t *next;
		boolean_t copy;
		int rc;

		ASSERT(stpp != NULL);

		/* Split stpp off */
		next = stpp->stp_next;
		stpp->stp_next = NULL;

		if (next != NULL)
			prefetch_read_many(next);

		if (stp->st_state != SFXGE_TXQ_STARTED)
			goto reject;

		copy = B_FALSE;

again:
		/* Fragment the packet */
		if (stpp->stp_mss != 0) {
			rc = sfxge_tx_qlso_fragment(stp, stpp, copy);
		} else {
			rc = sfxge_tx_qpacket_fragment(stp, stpp, copy);
		}

		switch (rc) {
		case 0:
			break;

		case ENOSPC:
			if (!copy)
				goto copy;

		/*FALLTHRU*/
		default:
			goto reject;
		}

		/* Free the packet structure */
		stpp->stp_etherhp = NULL;
		stpp->stp_iphp = NULL;
		stpp->stp_thp = NULL;
		stpp->stp_off = 0;
		stpp->stp_size = 0;
		stpp->stp_mss = 0;
		stpp->stp_dpl_put_len = 0;

		ASSERT3P(stpp->stp_mp, ==, NULL);

		if (sfxge_tx_qfpp_put(stp, stpp) != 0) {
			sfxge_tx_packet_destroy(sp, stpp);
			stpp = NULL;
		}

		--count;
		stpp = next;

		/* Post the packet */
		sfxge_tx_qlist_post(stp);

		if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED)
			goto defer;

		if (stp->st_added - pushed >= SFXGE_TX_BATCH) {
			efx_tx_qpush(stp->st_etp, stp->st_added, pushed);
			pushed = stp->st_added;
		}

		continue;

copy:
		/* Abort the current fragment list */
		sfxge_tx_qlist_abort(stp);

		/* Try copying the packet to flatten it */
		ASSERT(!copy);
		copy = B_TRUE;

		goto again;

reject:
		/* Abort the current fragment list */
		sfxge_tx_qlist_abort(stp);

		/* Discard the packet */
		freemsg(stpp->stp_mp);
		stpp->stp_mp = NULL;

		/* Free the packet structure */
		stpp->stp_etherhp = NULL;
		stpp->stp_iphp = NULL;
		stpp->stp_thp = NULL;
		stpp->stp_off = 0;
		stpp->stp_size = 0;
		stpp->stp_mss = 0;
		stpp->stp_dpl_put_len = 0;

		if (sfxge_tx_qfpp_put(stp, stpp) != 0) {
			sfxge_tx_packet_destroy(sp, stpp);
			stpp = NULL;
		}

		--count;
		stpp = next;
		continue;
defer:
		DTRACE_PROBE1(defer, unsigned int, stp->st_index);
		break;
	}

	if (count == 0) {
		/* New empty get list */
		ASSERT3P(stpp, ==, NULL);
		stdp->std_get = NULL;
		stdp->std_count = 0;

		stdp->std_getp = &(stdp->std_get);
	} else {
		/* shorten the list by moving the head */
		stdp->std_get = stpp;
		stdp->std_count = count;
		ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp));
	}

	if (stp->st_added != pushed)
		efx_tx_qpush(stp->st_etp, stp->st_added, pushed);

	ASSERT(stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED ||
	    stdp->std_count == 0);
}

/* Swizzle deferred packet list, try and push to HW */
static inline void
sfxge_tx_qdpl_service(sfxge_txq_t *stp)
{
	do {
		ASSERT(mutex_owned(&(stp->st_lock)));

		if (SFXGE_TX_QDPL_PUT_PENDING(stp))
			sfxge_tx_qdpl_swizzle(stp);

		if (stp->st_unblock == SFXGE_TXQ_NOT_BLOCKED)
			sfxge_tx_qdpl_drain(stp);

		mutex_exit(&(stp->st_lock));

		if (!SFXGE_TX_QDPL_PUT_PENDING(stp))
			break;
	} while (mutex_tryenter(&(stp->st_lock)));
}

static void
sfxge_tx_qdpl_flush_locked(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
	sfxge_tx_packet_t *stpp;
	unsigned int count;

	ASSERT(mutex_owned(&(stp->st_lock)));

	/* Swizzle put list to the get list */
	sfxge_tx_qdpl_swizzle(stp);

	stpp = stdp->std_get;
	count = stdp->std_count;

	while (count != 0) {
		sfxge_tx_packet_t *next;

		next = stpp->stp_next;
		stpp->stp_next = NULL;

		/* Discard the packet */
		freemsg(stpp->stp_mp);
		stpp->stp_mp = NULL;

		/* Free the packet structure */
		stpp->stp_etherhp = NULL;
		stpp->stp_iphp = NULL;
		stpp->stp_thp = NULL;
		stpp->stp_off = 0;
		stpp->stp_size = 0;
		stpp->stp_mss = 0;
		stpp->stp_dpl_put_len = 0;

		sfxge_tx_packet_destroy(sp, stpp);

		--count;
		stpp = next;
	}

	ASSERT3P(stpp, ==, NULL);

	/* Empty list */
	stdp->std_get = NULL;
	stdp->std_count = 0;
	stdp->std_getp = &(stdp->std_get);
}


void
sfxge_tx_qdpl_flush(sfxge_txq_t *stp)
{
	mutex_enter(&(stp->st_lock));
	sfxge_tx_qdpl_flush_locked(stp);
	mutex_exit(&(stp->st_lock));
}


static void
sfxge_tx_qunblock(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	unsigned int evq = stp->st_evq;
	sfxge_evq_t *sep = sp->s_sep[evq];

	ASSERT(mutex_owned(&(sep->se_lock)));

	mutex_enter(&(stp->st_lock));

	if (stp->st_state != SFXGE_TXQ_STARTED) {
		mutex_exit(&(stp->st_lock));
		return;
	}

	if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED) {
		unsigned int level;

		level = stp->st_added - stp->st_completed;
		if (level <= stp->st_unblock) {
			stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;
			sfxge_tx_qlist_post(stp);
		}
	}

	sfxge_tx_qdpl_service(stp);
	/* lock has been dropped */
}

void
sfxge_tx_qcomplete(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);
	unsigned int evq = stp->st_evq;
	sfxge_evq_t *sep = sp->s_sep[evq];
	unsigned int completed;

	ASSERT(mutex_owned(&(sep->se_lock)));

	completed = stp->st_completed;
	while (completed != stp->st_pending) {
		unsigned int id;
		sfxge_tx_mapping_t *stmp;

		id = completed++ & (SFXGE_TX_NDESCS - 1);

		if ((stmp = stp->st_stmp[id]) != NULL) {
			mblk_t *mp;

			/* Unbind all the mappings */
			do {
				ASSERT(stmp->stm_mp != NULL);
				sfxge_tx_msgb_unbind(stmp);

				stmp = stmp->stm_next;
			} while (stmp != NULL);

			/*
			 * Now that the packet is no longer mapped for DMA it
			 * can be freed.
			 */
			mp = stp->st_mp[id];
			stp->st_mp[id] = NULL;

			ASSERT(mp != NULL);
			freemsg(mp);
		}
	}
	stp->st_completed = completed;

	/* Check whether we need to unblock the queue */
	if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED) {
		unsigned int level;

		level = stp->st_added - stp->st_completed;
		if (level <= stp->st_unblock)
			sfxge_tx_qunblock(stp);
	}

	/* Release TX backpressure from the TX DPL put/get list being full */
	if (stdp->std_count < stdp->get_pkt_limit)
		mac_tx_update(sp->s_mh);
}

void
sfxge_tx_qflush_done(sfxge_txq_t *stp)
{
	sfxge_t *sp = stp->st_sp;
	boolean_t flush_pending = B_FALSE;

	ASSERT(mutex_owned(&(sp->s_sep[stp->st_evq]->se_lock)));

	mutex_enter(&(stp->st_lock));

	switch (stp->st_state) {
	case SFXGE_TXQ_INITIALIZED:
		/* Ignore flush event after TxQ destroyed */
		break;

	case SFXGE_TXQ_FLUSH_PENDING:
		flush_pending = B_TRUE;
		stp->st_state = SFXGE_TXQ_FLUSH_DONE;
		break;

	case SFXGE_TXQ_FLUSH_FAILED:
		/* MC may have rebooted before handling the flush request */
		stp->st_state = SFXGE_TXQ_FLUSH_DONE;
		break;

	case SFXGE_TXQ_STARTED:
		/*
		 * MC initiated flush on MC reboot or because of bad Tx
		 * descriptor
		 */
		stp->st_state = SFXGE_TXQ_FLUSH_DONE;
		break;

	case SFXGE_TXQ_FLUSH_DONE:
		/* Ignore unexpected extra flush event */
		ASSERT(B_FALSE);
		break;

	default:
		ASSERT(B_FALSE);
	}


	mutex_exit(&(stp->st_lock));

	if (flush_pending == B_FALSE) {
		/* Flush was not pending */
		return;
	}

	mutex_enter(&(sp->s_tx_flush_lock));
	sp->s_tx_flush_pending--;
	if (sp->s_tx_flush_pending <= 0) {
		/* All queues flushed: wakeup sfxge_tx_stop() */
		cv_signal(&(sp->s_tx_flush_kv));
	}
	mutex_exit(&(sp->s_tx_flush_lock));
}

static void
sfxge_tx_qflush(sfxge_t *sp, unsigned int index, boolean_t wait_for_flush)
{
	sfxge_txq_t *stp = sp->s_stp[index];
	int rc;

	ASSERT(mutex_owned(&(sp->s_state_lock)));
	ASSERT(mutex_owned(&(sp->s_tx_flush_lock)));

	mutex_enter(&(stp->st_lock));

	/* Prepare to flush and stop the queue */
	if (stp->st_state == SFXGE_TXQ_STARTED) {
		/* Flush the transmit queue */
		if ((rc = efx_tx_qflush(stp->st_etp)) == EALREADY) {
			/* Already flushed, may be initiated by MC */
			stp->st_state = SFXGE_TXQ_FLUSH_DONE;
		} else if (rc != 0) {
			/* Unexpected error */
			stp->st_state = SFXGE_TXQ_FLUSH_FAILED;
		} else if (wait_for_flush) {
			stp->st_state = SFXGE_TXQ_FLUSH_PENDING;
			sp->s_tx_flush_pending++;
		} else {
			/* Assume the flush is done */
			stp->st_state = SFXGE_TXQ_FLUSH_DONE;
		}
	}

	mutex_exit(&(stp->st_lock));
}

static void
sfxge_tx_qstop(sfxge_t *sp, unsigned int index)
{
	sfxge_txq_t *stp = sp->s_stp[index];
	unsigned int evq = stp->st_evq;
	sfxge_evq_t *sep = sp->s_sep[evq];

	mutex_enter(&(sep->se_lock));
	mutex_enter(&(stp->st_lock));

	if (stp->st_state == SFXGE_TXQ_INITIALIZED)
		goto done;

	ASSERT(stp->st_state == SFXGE_TXQ_FLUSH_PENDING ||
	    stp->st_state == SFXGE_TXQ_FLUSH_DONE ||
	    stp->st_state == SFXGE_TXQ_FLUSH_FAILED);

	/* All queues should have been flushed */
	if (stp->st_sp->s_tx_flush_pending != 0) {
		dev_err(sp->s_dip, CE_NOTE,
		    SFXGE_CMN_ERR "txq[%d] stop with flush_pending=%d",
		    index, stp->st_sp->s_tx_flush_pending);
	}
	if (stp->st_state == SFXGE_TXQ_FLUSH_FAILED) {
		dev_err(sp->s_dip, CE_NOTE,
		    SFXGE_CMN_ERR "txq[%d] flush failed", index);
	}

	/* Destroy the transmit queue */
	efx_tx_qdestroy(stp->st_etp);
	stp->st_etp = NULL;

	/* Clear entries from the buffer table */
	sfxge_sram_buf_tbl_clear(sp, stp->st_id,
	    EFX_TXQ_NBUFS(SFXGE_TX_NDESCS));

	sfxge_tx_qlist_abort(stp);
	ASSERT3U(stp->st_n, ==, 0);

	stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED;

	stp->st_pending = stp->st_added;

	sfxge_tx_qcomplete(stp);
	ASSERT3U(stp->st_completed, ==, stp->st_pending);

	sfxge_tx_qreap(stp);
	ASSERT3U(stp->st_reaped, ==, stp->st_completed);

	/*
	 * Ensure the deferred packet list is cleared
	 * Can race with sfxge_tx_packet_add() adding to the put list
	 */
	sfxge_tx_qdpl_flush_locked(stp);

	stp->st_added = 0;
	stp->st_pending = 0;
	stp->st_completed = 0;
	stp->st_reaped = 0;

	stp->st_state = SFXGE_TXQ_INITIALIZED;

done:
	mutex_exit(&(stp->st_lock));
	mutex_exit(&(sep->se_lock));
}

static void
sfxge_tx_qfini(sfxge_t *sp, unsigned int index)
{
	sfxge_txq_t *stp = sp->s_stp[index];
	sfxge_tx_dpl_t *stdp = &(stp->st_dpl);

	ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED);
	stp->st_state = SFXGE_TXQ_UNINITIALIZED;

	/* Detach the TXQ from the driver */
	sp->s_stp[index] = NULL;
	ASSERT(sp->s_tx_qcount > 0);
	sp->s_tx_qcount--;

	/* Free the EVQ label for events from this TXQ */
	(void) sfxge_ev_txlabel_free(sp, stp->st_evq, stp, stp->st_label);
	stp->st_label = 0;

	/* Tear down the statistics */
	sfxge_tx_kstat_fini(stp);

	/* Ensure the deferred packet list is empty */
	ASSERT3U(stdp->std_count, ==, 0);
	ASSERT3P(stdp->std_get, ==, NULL);
	ASSERT3U(stdp->std_put, ==, 0);

	/* Clear the free buffer pool */
	sfxge_tx_qfbp_empty(stp);

	/* Clear the free mapping pool */
	sfxge_tx_qfmp_empty(stp);

	/* Clear the free packet pool */
	sfxge_tx_qfpp_empty(stp);

	mutex_destroy(&(stp->st_lock));

	stp->st_evq = 0;
	stp->st_type = 0;
	stp->st_index = 0;

	kmem_cache_free(sp->s_tqc, stp);
}

int
sfxge_tx_init(sfxge_t *sp)
{
	sfxge_intr_t *sip = &(sp->s_intr);
	char name[MAXNAMELEN];
	sfxge_txq_type_t qtype;
	unsigned int txq, evq;
	int index;
	int rc;

	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_tx_packet_cache",
	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));

	sp->s_tpc = kmem_cache_create(name, sizeof (sfxge_tx_packet_t),
	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_packet_ctor, sfxge_tx_packet_dtor,
	    NULL, sp, NULL, 0);
	ASSERT(sp->s_tpc != NULL);

	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_tx_buffer_cache",
	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));

	sp->s_tbc = kmem_cache_create(name, sizeof (sfxge_tx_buffer_t),
	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_buffer_ctor, sfxge_tx_buffer_dtor,
	    NULL, sp, NULL, 0);
	ASSERT(sp->s_tbc != NULL);

	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_tx_mapping_cache",
	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));

	sp->s_tmc = kmem_cache_create(name, sizeof (sfxge_tx_mapping_t),
	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_mapping_ctor, sfxge_tx_mapping_dtor,
	    NULL, sp, NULL, 0);
	ASSERT(sp->s_tmc != NULL);

	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_txq_cache",
	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));

	sp->s_tqc = kmem_cache_create(name, sizeof (sfxge_txq_t),
	    SFXGE_CPU_CACHE_SIZE, sfxge_tx_qctor, sfxge_tx_qdtor, NULL, sp,
	    NULL, 0);
	ASSERT(sp->s_tqc != NULL);

	/* Initialize the transmit queues. */
	sp->s_tx_scale_max[SFXGE_TXQ_NON_CKSUM]		= sip->si_nalloc;
	sp->s_tx_scale_max[SFXGE_TXQ_IP_CKSUM]		= 1;
	sp->s_tx_scale_max[SFXGE_TXQ_IP_TCP_UDP_CKSUM]	= sip->si_nalloc;

	/* Ensure minimum queue counts required by sfxge_tx_packet_add(). */
	if (sp->s_tx_scale_max[SFXGE_TXQ_NON_CKSUM] < 1)
		sp->s_tx_scale_max[SFXGE_TXQ_NON_CKSUM] = 1;

	if (sp->s_tx_scale_max[SFXGE_TXQ_IP_CKSUM] < 1)
		sp->s_tx_scale_max[SFXGE_TXQ_IP_CKSUM] = 1;

	txq = 0;
	for (qtype = 0; qtype < SFXGE_TXQ_NTYPES; qtype++) {
		unsigned int tx_scale = sp->s_tx_scale_max[qtype];

		if (txq + tx_scale > EFX_ARRAY_SIZE(sp->s_stp)) {
			rc = EINVAL;
			goto fail1;
		}

		sp->s_tx_scale_base[qtype] = txq;

		for (evq = 0; evq < tx_scale; evq++) {
			if ((rc = sfxge_tx_qinit(sp, txq, qtype, evq)) != 0) {
				goto fail2;
			}
			txq++;
		}
		ASSERT3U(txq, <=, EFX_ARRAY_SIZE(sp->s_stp));
	}

	return (0);

fail2:
	DTRACE_PROBE(fail2);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	index = EFX_ARRAY_SIZE(sp->s_stp);
	while (--index >= 0) {
		if (sp->s_stp[index] != NULL)
			sfxge_tx_qfini(sp, index);
	}

	kmem_cache_destroy(sp->s_tqc);
	sp->s_tqc = NULL;

	kmem_cache_destroy(sp->s_tmc);
	sp->s_tmc = NULL;

	kmem_cache_destroy(sp->s_tbc);
	sp->s_tbc = NULL;

	kmem_cache_destroy(sp->s_tpc);
	sp->s_tpc = NULL;

	return (rc);
}

int
sfxge_tx_start(sfxge_t *sp)
{
	efx_nic_t *enp = sp->s_enp;
	int index;
	int rc;

	/* Initialize the transmit module */
	if ((rc = efx_tx_init(enp)) != 0)
		goto fail1;

	for (index = 0; index < EFX_ARRAY_SIZE(sp->s_stp); index++) {
		if (sp->s_stp[index] != NULL)
			if ((rc = sfxge_tx_qstart(sp, index)) != 0)
				goto fail2;
	}

	return (0);

fail2:
	DTRACE_PROBE(fail2);

	sfxge_tx_stop(sp);

fail1:
	DTRACE_PROBE1(fail1, int, rc);

	return (rc);
}


/*
 * Add a packet to the TX Deferred Packet List and if the TX queue lock
 * can be acquired then call sfxge_tx_qdpl_service() to fragment and push
 * to the H/W transmit descriptor ring
 *
 * If ENOSPC is returned then the DPL is full or the packet create failed, but
 * the mblk isn't freed so that the caller can return this mblk from mc_tx() to
 * back-pressure the OS stack.
 *
 * For all other errors the mblk is freed
 */
int
sfxge_tx_packet_add(sfxge_t *sp, mblk_t *mp)
{
	struct ether_header *etherhp;
	struct ip *iphp;
	struct tcphdr *thp;
	size_t off;
	size_t size;
	size_t mss;
	sfxge_txq_t *stp;
	unsigned int txq;
	int index;
	boolean_t locked;
	sfxge_tx_packet_t *stpp;
	sfxge_packet_type_t pkt_type;
	uint16_t sport, dport;
	int rc = 0;

	ASSERT3P(mp->b_next, ==, NULL);
	ASSERT(!(DB_CKSUMFLAGS(mp) & HCK_PARTIALCKSUM));

	/*
	 * Do not enqueue packets during startup/shutdown;
	 *
	 * NOTE: This access to the state is NOT protected by the state lock. It
	 * is an imperfect test and anything further getting onto the get/put
	 * deferred packet lists is cleaned up in (possibly repeated) calls to
	 * sfxge_can_destroy().
	 */
	if (sp->s_state != SFXGE_STARTED) {
		rc = EINVAL;
		goto fail1;
	}

	etherhp = NULL;
	iphp = NULL;
	thp = NULL;
	off = 0;
	size = 0;
	mss = 0;

	/* Check whether we need the header pointers for LSO segmentation */
	if (DB_LSOFLAGS(mp) & HW_LSO) {
		/* LSO segmentation relies on hardware checksum offload */
		DB_CKSUMFLAGS(mp) |= HCK_FULLCKSUM;

		if ((mss = DB_LSOMSS(mp)) == 0) {
			rc = EINVAL;
			goto fail1;
		}

		pkt_type = sfxge_pkthdr_parse(mp, &etherhp, &iphp, &thp,
		    &off, &size, &sport, &dport);

		if (pkt_type != SFXGE_PACKET_TYPE_IPV4_TCP ||
		    etherhp == NULL ||
		    iphp == NULL ||
		    thp == NULL ||
		    off == 0) {
			rc = EINVAL;
			goto fail2;
		}
	}

	/* Choose the appropriate transit queue */
	if (DB_CKSUMFLAGS(mp) & HCK_FULLCKSUM) {
		sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);

		if (srsp->srs_state == SFXGE_RX_SCALE_STARTED) {
			uint32_t hash;

			if (srsp->srs_count > 1) {
				/*
				 * If we have not already parsed the headers
				 * for LSO segmentation then we need to do it
				 * now so we can calculate the hash.
				 */
				if (thp == NULL) {
					(void) sfxge_pkthdr_parse(mp, &etherhp,
					    &iphp, &thp, &off, &size,
					    &sport, &dport);
				}

				if (thp != NULL) {
					SFXGE_TCP_HASH(sp,
					    &iphp->ip_dst.s_addr,
					    thp->th_dport,
					    &iphp->ip_src.s_addr,
					    thp->th_sport, hash);

					index = srsp->srs_tbl[hash %
					    SFXGE_RX_SCALE_MAX];
				} else if (iphp != NULL) {
					/*
					 * Calculate IPv4 4-tuple hash, with
					 * TCP/UDP/SCTP src/dest ports. Ports
					 * are zero for other IPv4 protocols.
					 */
					SFXGE_IP_HASH(sp,
					    &iphp->ip_dst.s_addr, dport,
					    &iphp->ip_src.s_addr, sport, hash);

					index = srsp->srs_tbl[hash %
					    SFXGE_RX_SCALE_MAX];
				} else {
					/*
					 * Other traffic always goes to the
					 * the queue in the zero-th entry of
					 * the RSS table.
					 */
					index = srsp->srs_tbl[0];
				}
			} else {
				/*
				 * It does not matter what the hash is
				 * because all the RSS table entries will be
				 * the same.
				 */
				index = srsp->srs_tbl[0];
			}

			/*
			 * Find the event queue corresponding to the hash in
			 * the RSS table.
			 */
			txq = sp->s_tx_scale_base[SFXGE_TXQ_IP_TCP_UDP_CKSUM] +
			    index;
			stp = sp->s_stp[txq];
			ASSERT3U(stp->st_evq, ==, index);
		} else {
			index = 0;
			txq = sp->s_tx_scale_base[SFXGE_TXQ_IP_TCP_UDP_CKSUM] +
			    index;
			stp = sp->s_stp[txq];
		}
	} else if (DB_CKSUMFLAGS(mp) & HCK_IPV4_HDRCKSUM) {
		ASSERT3U(sp->s_tx_scale_max[SFXGE_TXQ_IP_CKSUM], >=, 1);
		index = 0;
		txq = sp->s_tx_scale_base[SFXGE_TXQ_IP_CKSUM] + index;
		stp = sp->s_stp[txq];
	} else {
		/*
		 * No hardware checksum offload requested.
		 */
		sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);

		if (srsp->srs_state == SFXGE_RX_SCALE_STARTED) {
			uint32_t hash = 0;

			if (srsp->srs_count > 1) {
				if (iphp == NULL) {
					(void) sfxge_pkthdr_parse(mp, &etherhp,
					    &iphp, &thp, &off, &size,
					    &sport, &dport);
				}

				if (iphp != NULL) {
					/*
					 * Calculate IPv4 4-tuple hash, with
					 * TCP/UDP/SCTP src/dest ports. Ports
					 * are zero for other IPv4 protocols.
					 */
					SFXGE_IP_HASH(sp,
					    &iphp->ip_dst.s_addr, dport,
					    &iphp->ip_src.s_addr, sport, hash);

					hash = hash % SFXGE_RX_SCALE_MAX;
				}
			}
			index = srsp->srs_tbl[hash];

			/*
			 * The RSS table (indexed by hash) gives the RXQ index,
			 * (mapped 1:1 with EVQs). Find the TXQ that results in
			 * using the same EVQ as for the RX data path.
			 */
			ASSERT3U(sp->s_tx_scale_max[SFXGE_TXQ_NON_CKSUM],
			    >, index);
			txq = sp->s_tx_scale_base[SFXGE_TXQ_NON_CKSUM] + index;
			stp = sp->s_stp[txq];
			ASSERT3U(stp->st_evq, ==, index);
		} else {
			ASSERT3U(sp->s_tx_scale_max[SFXGE_TXQ_NON_CKSUM], >, 0);
			index = 0;
			txq = sp->s_tx_scale_base[SFXGE_TXQ_NON_CKSUM] + index;
			stp = sp->s_stp[txq];
		}


	}
	ASSERT(stp != NULL);

	ASSERT(mss == 0 || (DB_LSOFLAGS(mp) & HW_LSO));

	/* Try to grab the lock */
	locked = mutex_tryenter(&(stp->st_lock));

	if (locked) {
		/* Try to grab a packet from the pool */
		stpp = sfxge_tx_qfpp_get(stp);
	} else {
		stpp = NULL;
	}

	if (stpp == NULL) {
		/*
		 * Either the pool was empty or we don't have the lock so
		 * allocate a new packet.
		 */
		if ((stpp = sfxge_tx_packet_create(sp)) == NULL) {
			rc = ENOSPC;
			goto fail3;
		}
	}

	stpp->stp_mp = mp;
	stpp->stp_etherhp = etherhp;
	stpp->stp_iphp = iphp;
	stpp->stp_thp = thp;
	stpp->stp_off = off;
	stpp->stp_size = size;
	stpp->stp_mss = mss;
	stpp->stp_dpl_put_len = 0;

	rc = sfxge_tx_qdpl_add(stp, stpp, locked);
	if (rc != 0) {
		/* ENOSPC can happen for DPL get or put list is full */
		ASSERT3U(rc, ==, ENOSPC);

		/*
		 * Note; if this is the unlocked DPL put list full case there is
		 * no need to worry about a race with locked
		 * sfxge_tx_qdpl_swizzle() as we know that the TX DPL put list
		 * was full and would have been swizzle'd to the TX DPL get
		 * list; hence guaranteeing future TX completions and calls
		 * to mac_tx_update() via sfxge_tx_qcomplete()
		 */
		goto fail4;
	}

	/* Try to grab the lock again */
	if (!locked)
		locked = mutex_tryenter(&(stp->st_lock));

	if (locked) {
		/* Try to service the list */
		sfxge_tx_qdpl_service(stp);
		/* lock has been dropped */
	}

	return (0);

fail4:
	DTRACE_PROBE(fail4);
	sfxge_tx_packet_destroy(sp, stpp);
fail3:
	DTRACE_PROBE(fail3);
	if (locked)
		mutex_exit(&(stp->st_lock));
fail2:
	DTRACE_PROBE(fail2);
fail1:
	DTRACE_PROBE1(fail1, int, rc);

	if (rc != ENOSPC)
		freemsg(mp);
	return (rc);
}

void
sfxge_tx_stop(sfxge_t *sp)
{
	efx_nic_t *enp = sp->s_enp;
	clock_t timeout;
	boolean_t wait_for_flush;
	int index;

	ASSERT(mutex_owned(&(sp->s_state_lock)));

	mutex_enter(&(sp->s_tx_flush_lock));

	/* Flush all the queues */
	if (sp->s_hw_err == SFXGE_HW_OK) {
		wait_for_flush = B_TRUE;
	} else {
		/*
		 * Flag indicates possible hardware failure.
		 * Attempt flush but do not wait for it to complete.
		 */
		wait_for_flush = B_FALSE;
	}

	/* Prepare queues to stop and flush the hardware ring */
	index = EFX_ARRAY_SIZE(sp->s_stp);
	while (--index >= 0) {
		if (sp->s_stp[index] != NULL)
			sfxge_tx_qflush(sp, index, wait_for_flush);
	}

	if (wait_for_flush == B_FALSE)
		goto flush_done;

	/* Wait upto 2sec for queue flushing to complete */
	timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_TX_QFLUSH_USEC);

	while (sp->s_tx_flush_pending > 0) {
		if (cv_timedwait(&(sp->s_tx_flush_kv), &(sp->s_tx_flush_lock),
		    timeout) < 0) {
			/* Timeout waiting for queues to flush */
			dev_info_t *dip = sp->s_dip;

			DTRACE_PROBE(timeout);
			dev_err(dip, CE_NOTE,
			    SFXGE_CMN_ERR "tx qflush timeout");
			break;
		}
	}

flush_done:
	sp->s_tx_flush_pending = 0;
	mutex_exit(&(sp->s_tx_flush_lock));

	/* Stop all the queues */
	index = EFX_ARRAY_SIZE(sp->s_stp);
	while (--index >= 0) {
		if (sp->s_stp[index] != NULL)
			sfxge_tx_qstop(sp, index);
	}

	/* Tear down the transmit module */
	efx_tx_fini(enp);
}

void
sfxge_tx_fini(sfxge_t *sp)
{
	int index;

	index = EFX_ARRAY_SIZE(sp->s_stp);
	while (--index >= 0) {
		if (sp->s_stp[index] != NULL)
			sfxge_tx_qfini(sp, index);
	}

	kmem_cache_destroy(sp->s_tqc);
	sp->s_tqc = NULL;

	kmem_cache_destroy(sp->s_tmc);
	sp->s_tmc = NULL;

	kmem_cache_destroy(sp->s_tbc);
	sp->s_tbc = NULL;

	kmem_cache_destroy(sp->s_tpc);
	sp->s_tpc = NULL;
}