xref: /onnv-gate/usr/src/uts/common/io/nxge/nxge_rxdma.c (revision 7812:f4514b3ad5a2)
13859Sml29623 /*
23859Sml29623  * CDDL HEADER START
33859Sml29623  *
43859Sml29623  * The contents of this file are subject to the terms of the
53859Sml29623  * Common Development and Distribution License (the "License").
63859Sml29623  * You may not use this file except in compliance with the License.
73859Sml29623  *
83859Sml29623  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
93859Sml29623  * or http://www.opensolaris.org/os/licensing.
103859Sml29623  * See the License for the specific language governing permissions
113859Sml29623  * and limitations under the License.
123859Sml29623  *
133859Sml29623  * When distributing Covered Code, include this CDDL HEADER in each
143859Sml29623  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
153859Sml29623  * If applicable, add the following below this CDDL HEADER, with the
163859Sml29623  * fields enclosed by brackets "[]" replaced with your own identifying
173859Sml29623  * information: Portions Copyright [yyyy] [name of copyright owner]
183859Sml29623  *
193859Sml29623  * CDDL HEADER END
203859Sml29623  */
213859Sml29623 /*
225759Smisaki  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
233859Sml29623  * Use is subject to license terms.
243859Sml29623  */
253859Sml29623 
263859Sml29623 #include <sys/nxge/nxge_impl.h>
273859Sml29623 #include <sys/nxge/nxge_rxdma.h>
286495Sspeer #include <sys/nxge/nxge_hio.h>
296495Sspeer 
306495Sspeer #if !defined(_BIG_ENDIAN)
316495Sspeer #include <npi_rx_rd32.h>
326495Sspeer #endif
336495Sspeer #include <npi_rx_rd64.h>
346495Sspeer #include <npi_rx_wr64.h>
353859Sml29623 
363859Sml29623 #define	NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp)	\
376495Sspeer 	(rdcgrp + nxgep->pt_config.hw_config.def_mac_rxdma_grpid)
383859Sml29623 #define	NXGE_ACTUAL_RDC(nxgep, rdc)	\
393859Sml29623 	(rdc + nxgep->pt_config.hw_config.start_rdc)
403859Sml29623 
413859Sml29623 /*
423859Sml29623  * Globals: tunable parameters (/etc/system or adb)
433859Sml29623  *
443859Sml29623  */
453859Sml29623 extern uint32_t nxge_rbr_size;
463859Sml29623 extern uint32_t nxge_rcr_size;
473859Sml29623 extern uint32_t	nxge_rbr_spare_size;
483859Sml29623 
493859Sml29623 extern uint32_t nxge_mblks_pending;
503859Sml29623 
513859Sml29623 /*
523859Sml29623  * Tunable to reduce the amount of time spent in the
533859Sml29623  * ISR doing Rx Processing.
543859Sml29623  */
553859Sml29623 extern uint32_t nxge_max_rx_pkts;
563859Sml29623 boolean_t nxge_jumbo_enable;
573859Sml29623 
583859Sml29623 /*
593859Sml29623  * Tunables to manage the receive buffer blocks.
603859Sml29623  *
613859Sml29623  * nxge_rx_threshold_hi: copy all buffers.
623859Sml29623  * nxge_rx_bcopy_size_type: receive buffer block size type.
633859Sml29623  * nxge_rx_threshold_lo: copy only up to tunable block size type.
643859Sml29623  */
653859Sml29623 extern nxge_rxbuf_threshold_t nxge_rx_threshold_hi;
663859Sml29623 extern nxge_rxbuf_type_t nxge_rx_buf_size_type;
673859Sml29623 extern nxge_rxbuf_threshold_t nxge_rx_threshold_lo;
683859Sml29623 
696611Sml29623 extern uint32_t	nxge_cksum_offload;
706495Sspeer 
716495Sspeer static nxge_status_t nxge_map_rxdma(p_nxge_t, int);
726495Sspeer static void nxge_unmap_rxdma(p_nxge_t, int);
733859Sml29623 
743859Sml29623 static nxge_status_t nxge_rxdma_hw_start_common(p_nxge_t);
756495Sspeer 
766495Sspeer static nxge_status_t nxge_rxdma_hw_start(p_nxge_t, int);
776495Sspeer static void nxge_rxdma_hw_stop(p_nxge_t, int);
783859Sml29623 
793859Sml29623 static nxge_status_t nxge_map_rxdma_channel(p_nxge_t, uint16_t,
803859Sml29623     p_nxge_dma_common_t *,  p_rx_rbr_ring_t *,
813859Sml29623     uint32_t,
823859Sml29623     p_nxge_dma_common_t *, p_rx_rcr_ring_t *,
833859Sml29623     p_rx_mbox_t *);
843859Sml29623 static void nxge_unmap_rxdma_channel(p_nxge_t, uint16_t,
853859Sml29623     p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t);
863859Sml29623 
873859Sml29623 static nxge_status_t nxge_map_rxdma_channel_cfg_ring(p_nxge_t,
883859Sml29623     uint16_t,
893859Sml29623     p_nxge_dma_common_t *, p_rx_rbr_ring_t *,
903859Sml29623     p_rx_rcr_ring_t *, p_rx_mbox_t *);
913859Sml29623 static void nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t,
923859Sml29623     p_rx_rcr_ring_t, p_rx_mbox_t);
933859Sml29623 
943859Sml29623 static nxge_status_t nxge_map_rxdma_channel_buf_ring(p_nxge_t,
953859Sml29623     uint16_t,
963859Sml29623     p_nxge_dma_common_t *,
973859Sml29623     p_rx_rbr_ring_t *, uint32_t);
983859Sml29623 static void nxge_unmap_rxdma_channel_buf_ring(p_nxge_t,
993859Sml29623     p_rx_rbr_ring_t);
1003859Sml29623 
1013859Sml29623 static nxge_status_t nxge_rxdma_start_channel(p_nxge_t, uint16_t,
1023859Sml29623     p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t);
1033859Sml29623 static nxge_status_t nxge_rxdma_stop_channel(p_nxge_t, uint16_t);
1043859Sml29623 
1056495Sspeer static mblk_t *
1066495Sspeer nxge_rx_pkts(p_nxge_t, p_rx_rcr_ring_t, rx_dma_ctl_stat_t, int);
1073859Sml29623 
1083859Sml29623 static void nxge_receive_packet(p_nxge_t,
1093859Sml29623 	p_rx_rcr_ring_t,
1103859Sml29623 	p_rcr_entry_t,
1113859Sml29623 	boolean_t *,
1123859Sml29623 	mblk_t **, mblk_t **);
1133859Sml29623 
1143859Sml29623 nxge_status_t nxge_disable_rxdma_channel(p_nxge_t, uint16_t);
1153859Sml29623 
1163859Sml29623 static p_rx_msg_t nxge_allocb(size_t, uint32_t, p_nxge_dma_common_t);
1173859Sml29623 static void nxge_freeb(p_rx_msg_t);
1186495Sspeer static void nxge_rx_pkts_vring(p_nxge_t, uint_t, rx_dma_ctl_stat_t);
1196495Sspeer static nxge_status_t nxge_rx_err_evnts(p_nxge_t, int, rx_dma_ctl_stat_t);
1203859Sml29623 
1213859Sml29623 static nxge_status_t nxge_rxdma_handle_port_errors(p_nxge_t,
1223859Sml29623 				uint32_t, uint32_t);
1233859Sml29623 
1243859Sml29623 static nxge_status_t nxge_rxbuf_index_info_init(p_nxge_t,
1253859Sml29623     p_rx_rbr_ring_t);
1263859Sml29623 
1273859Sml29623 
1283859Sml29623 static nxge_status_t
1293859Sml29623 nxge_rxdma_fatal_err_recover(p_nxge_t, uint16_t);
1303859Sml29623 
1313859Sml29623 nxge_status_t
1323859Sml29623 nxge_rx_port_fatal_err_recover(p_nxge_t);
1333859Sml29623 
1346495Sspeer static void nxge_rxdma_databuf_free(p_rx_rbr_ring_t);
1356495Sspeer 
1363859Sml29623 nxge_status_t
1373859Sml29623 nxge_init_rxdma_channels(p_nxge_t nxgep)
1383859Sml29623 {
1396495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
1406495Sspeer 	int i, count;
1413859Sml29623 
1423859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_rxdma_channels"));
1433859Sml29623 
1446495Sspeer 	if (!isLDOMguest(nxgep)) {
1456495Sspeer 		if (nxge_rxdma_hw_start_common(nxgep) != NXGE_OK) {
1466495Sspeer 			cmn_err(CE_NOTE, "hw_start_common");
1476495Sspeer 			return (NXGE_ERROR);
1486495Sspeer 		}
1496495Sspeer 	}
1506495Sspeer 
1516495Sspeer 	/*
1526495Sspeer 	 * NXGE_LOGICAL_GROUP_MAX > NXGE_MAX_RDC_GROUPS (8)
1536495Sspeer 	 * We only have 8 hardware RDC tables, but we may have
1546495Sspeer 	 * up to 16 logical (software-defined) groups of RDCS,
1556495Sspeer 	 * if we make use of layer 3 & 4 hardware classification.
1566495Sspeer 	 */
1576495Sspeer 	for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
1586495Sspeer 		if ((1 << i) & set->lg.map) {
1596495Sspeer 			int channel;
1606495Sspeer 			nxge_grp_t *group = set->group[i];
1616495Sspeer 			for (channel = 0; channel < NXGE_MAX_RDCS; channel++) {
1626495Sspeer 				if ((1 << channel) & group->map) {
1636495Sspeer 					if ((nxge_grp_dc_add(nxgep,
1647755SMisaki.Kataoka@Sun.COM 					    group, VP_BOUND_RX, channel)))
1656495Sspeer 						return (NXGE_ERROR);
1666495Sspeer 				}
1676495Sspeer 			}
1686495Sspeer 		}
1696495Sspeer 		if (++count == set->lg.count)
1706495Sspeer 			break;
1716495Sspeer 	}
1726495Sspeer 
1736495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_rxdma_channels"));
1746495Sspeer 
1756495Sspeer 	return (NXGE_OK);
1766495Sspeer }
1776495Sspeer 
1786495Sspeer nxge_status_t
1796495Sspeer nxge_init_rxdma_channel(p_nxge_t nxge, int channel)
1806495Sspeer {
1816495Sspeer 	nxge_status_t status;
1826495Sspeer 
1836495Sspeer 	NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_rxdma_channel"));
1846495Sspeer 
1856495Sspeer 	status = nxge_map_rxdma(nxge, channel);
1863859Sml29623 	if (status != NXGE_OK) {
1876495Sspeer 		NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
1886495Sspeer 		    "<== nxge_init_rxdma: status 0x%x", status));
1893859Sml29623 		return (status);
1903859Sml29623 	}
1913859Sml29623 
1926495Sspeer 	status = nxge_rxdma_hw_start(nxge, channel);
1933859Sml29623 	if (status != NXGE_OK) {
1946495Sspeer 		nxge_unmap_rxdma(nxge, channel);
1953859Sml29623 	}
1963859Sml29623 
1976495Sspeer 	if (!nxge->statsp->rdc_ksp[channel])
1986495Sspeer 		nxge_setup_rdc_kstats(nxge, channel);
1996495Sspeer 
2006495Sspeer 	NXGE_DEBUG_MSG((nxge, MEM2_CTL,
2016495Sspeer 	    "<== nxge_init_rxdma_channel: status 0x%x", status));
2023859Sml29623 
2033859Sml29623 	return (status);
2043859Sml29623 }
2053859Sml29623 
2063859Sml29623 void
2073859Sml29623 nxge_uninit_rxdma_channels(p_nxge_t nxgep)
2083859Sml29623 {
2096495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
2106495Sspeer 	int rdc;
2116495Sspeer 
2123859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channels"));
2133859Sml29623 
2146495Sspeer 	if (set->owned.map == 0) {
2156495Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
2166495Sspeer 		    "nxge_uninit_rxdma_channels: no channels"));
2176495Sspeer 		return;
2186495Sspeer 	}
2196495Sspeer 
2206495Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
2216495Sspeer 		if ((1 << rdc) & set->owned.map) {
2226495Sspeer 			nxge_grp_dc_remove(nxgep, VP_BOUND_RX, rdc);
2236495Sspeer 		}
2246495Sspeer 	}
2256495Sspeer 
2266495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_rxdma_channels"));
2276495Sspeer }
2286495Sspeer 
2296495Sspeer void
2306495Sspeer nxge_uninit_rxdma_channel(p_nxge_t nxgep, int channel)
2316495Sspeer {
2326495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channel"));
2336495Sspeer 
2346495Sspeer 	if (nxgep->statsp->rdc_ksp[channel]) {
2356495Sspeer 		kstat_delete(nxgep->statsp->rdc_ksp[channel]);
2366495Sspeer 		nxgep->statsp->rdc_ksp[channel] = 0;
2376495Sspeer 	}
2386495Sspeer 
2396495Sspeer 	nxge_rxdma_hw_stop(nxgep, channel);
2406495Sspeer 	nxge_unmap_rxdma(nxgep, channel);
2416495Sspeer 
2426495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uinit_rxdma_channel"));
2433859Sml29623 }
2443859Sml29623 
2453859Sml29623 nxge_status_t
2463859Sml29623 nxge_reset_rxdma_channel(p_nxge_t nxgep, uint16_t channel)
2473859Sml29623 {
2483859Sml29623 	npi_handle_t		handle;
2493859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
2503859Sml29623 	nxge_status_t		status = NXGE_OK;
2513859Sml29623 
252*7812SMichael.Speer@Sun.COM 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_reset_rxdma_channel"));
2533859Sml29623 
2543859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
2553859Sml29623 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
2563859Sml29623 
2573859Sml29623 	if (rs != NPI_SUCCESS) {
2583859Sml29623 		status = NXGE_ERROR | rs;
2593859Sml29623 	}
2603859Sml29623 
261*7812SMichael.Speer@Sun.COM 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_reset_rxdma_channel"));
262*7812SMichael.Speer@Sun.COM 
2633859Sml29623 	return (status);
2643859Sml29623 }
2653859Sml29623 
2663859Sml29623 void
2673859Sml29623 nxge_rxdma_regs_dump_channels(p_nxge_t nxgep)
2683859Sml29623 {
2696495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
2706495Sspeer 	int rdc;
2713859Sml29623 
2723859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_regs_dump_channels"));
2733859Sml29623 
2746495Sspeer 	if (!isLDOMguest(nxgep)) {
2756495Sspeer 		npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep);
2766495Sspeer 		(void) npi_rxdma_dump_fzc_regs(handle);
2776495Sspeer 	}
2786495Sspeer 
2796495Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
2806495Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
2816495Sspeer 		    "nxge_rxdma_regs_dump_channels: "
2826495Sspeer 		    "NULL ring pointer(s)"));
2833859Sml29623 		return;
2843859Sml29623 	}
2856495Sspeer 
2866495Sspeer 	if (set->owned.map == 0) {
2873859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
2886495Sspeer 		    "nxge_rxdma_regs_dump_channels: no channels"));
2893859Sml29623 		return;
2903859Sml29623 	}
2913859Sml29623 
2926495Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
2936495Sspeer 		if ((1 << rdc) & set->owned.map) {
2946495Sspeer 			rx_rbr_ring_t *ring =
2956495Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
2966495Sspeer 			if (ring) {
2976495Sspeer 				(void) nxge_dump_rxdma_channel(nxgep, rdc);
2986495Sspeer 			}
2993859Sml29623 		}
3003859Sml29623 	}
3013859Sml29623 
3023859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_regs_dump"));
3033859Sml29623 }
3043859Sml29623 
3053859Sml29623 nxge_status_t
3063859Sml29623 nxge_dump_rxdma_channel(p_nxge_t nxgep, uint8_t channel)
3073859Sml29623 {
3083859Sml29623 	npi_handle_t		handle;
3093859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
3103859Sml29623 	nxge_status_t		status = NXGE_OK;
3113859Sml29623 
3123859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_dump_rxdma_channel"));
3133859Sml29623 
3143859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
3153859Sml29623 	rs = npi_rxdma_dump_rdc_regs(handle, channel);
3163859Sml29623 
3173859Sml29623 	if (rs != NPI_SUCCESS) {
3183859Sml29623 		status = NXGE_ERROR | rs;
3193859Sml29623 	}
3203859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_dump_rxdma_channel"));
3213859Sml29623 	return (status);
3223859Sml29623 }
3233859Sml29623 
3243859Sml29623 nxge_status_t
3253859Sml29623 nxge_init_rxdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel,
3263859Sml29623     p_rx_dma_ent_msk_t mask_p)
3273859Sml29623 {
3283859Sml29623 	npi_handle_t		handle;
3293859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
3303859Sml29623 	nxge_status_t		status = NXGE_OK;
3313859Sml29623 
3323859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
3336929Smisaki 	    "<== nxge_init_rxdma_channel_event_mask"));
3343859Sml29623 
3353859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
3363859Sml29623 	rs = npi_rxdma_event_mask(handle, OP_SET, channel, mask_p);
3373859Sml29623 	if (rs != NPI_SUCCESS) {
3383859Sml29623 		status = NXGE_ERROR | rs;
3393859Sml29623 	}
3403859Sml29623 
3413859Sml29623 	return (status);
3423859Sml29623 }
3433859Sml29623 
3443859Sml29623 nxge_status_t
3453859Sml29623 nxge_init_rxdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel,
3463859Sml29623     p_rx_dma_ctl_stat_t cs_p)
3473859Sml29623 {
3483859Sml29623 	npi_handle_t		handle;
3493859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
3503859Sml29623 	nxge_status_t		status = NXGE_OK;
3513859Sml29623 
3523859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
3536929Smisaki 	    "<== nxge_init_rxdma_channel_cntl_stat"));
3543859Sml29623 
3553859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
3563859Sml29623 	rs = npi_rxdma_control_status(handle, OP_SET, channel, cs_p);
3573859Sml29623 
3583859Sml29623 	if (rs != NPI_SUCCESS) {
3593859Sml29623 		status = NXGE_ERROR | rs;
3603859Sml29623 	}
3613859Sml29623 
3623859Sml29623 	return (status);
3633859Sml29623 }
3643859Sml29623 
3656495Sspeer /*
3666495Sspeer  * nxge_rxdma_cfg_rdcgrp_default_rdc
3676495Sspeer  *
3686495Sspeer  *	Set the default RDC for an RDC Group (Table)
3696495Sspeer  *
3706495Sspeer  * Arguments:
3716495Sspeer  * 	nxgep
3726495Sspeer  *	rdcgrp	The group to modify
3736495Sspeer  *	rdc	The new default RDC.
3746495Sspeer  *
3756495Sspeer  * Notes:
3766495Sspeer  *
3776495Sspeer  * NPI/NXGE function calls:
3786495Sspeer  *	npi_rxdma_cfg_rdc_table_default_rdc()
3796495Sspeer  *
3806495Sspeer  * Registers accessed:
3816495Sspeer  *	RDC_TBL_REG: FZC_ZCP + 0x10000
3826495Sspeer  *
3836495Sspeer  * Context:
3846495Sspeer  *	Service domain
3856495Sspeer  */
3863859Sml29623 nxge_status_t
3876495Sspeer nxge_rxdma_cfg_rdcgrp_default_rdc(
3886495Sspeer 	p_nxge_t nxgep,
3896495Sspeer 	uint8_t rdcgrp,
3906495Sspeer 	uint8_t rdc)
3913859Sml29623 {
3923859Sml29623 	npi_handle_t		handle;
3933859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
3943859Sml29623 	p_nxge_dma_pt_cfg_t	p_dma_cfgp;
3953859Sml29623 	p_nxge_rdc_grp_t	rdc_grp_p;
3963859Sml29623 	uint8_t actual_rdcgrp, actual_rdc;
3973859Sml29623 
3983859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
3996929Smisaki 	    " ==> nxge_rxdma_cfg_rdcgrp_default_rdc"));
4003859Sml29623 	p_dma_cfgp = (p_nxge_dma_pt_cfg_t)&nxgep->pt_config;
4013859Sml29623 
4023859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
4033859Sml29623 
4046495Sspeer 	/*
4056495Sspeer 	 * This has to be rewritten.  Do we even allow this anymore?
4066495Sspeer 	 */
4073859Sml29623 	rdc_grp_p = &p_dma_cfgp->rdc_grps[rdcgrp];
4086495Sspeer 	RDC_MAP_IN(rdc_grp_p->map, rdc);
4096495Sspeer 	rdc_grp_p->def_rdc = rdc;
4103859Sml29623 
4113859Sml29623 	actual_rdcgrp = NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp);
4123859Sml29623 	actual_rdc = NXGE_ACTUAL_RDC(nxgep, rdc);
4133859Sml29623 
4146495Sspeer 	rs = npi_rxdma_cfg_rdc_table_default_rdc(
4156929Smisaki 	    handle, actual_rdcgrp, actual_rdc);
4163859Sml29623 
4173859Sml29623 	if (rs != NPI_SUCCESS) {
4183859Sml29623 		return (NXGE_ERROR | rs);
4193859Sml29623 	}
4203859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
4216929Smisaki 	    " <== nxge_rxdma_cfg_rdcgrp_default_rdc"));
4223859Sml29623 	return (NXGE_OK);
4233859Sml29623 }
4243859Sml29623 
4253859Sml29623 nxge_status_t
4263859Sml29623 nxge_rxdma_cfg_port_default_rdc(p_nxge_t nxgep, uint8_t port, uint8_t rdc)
4273859Sml29623 {
4283859Sml29623 	npi_handle_t		handle;
4293859Sml29623 
4303859Sml29623 	uint8_t actual_rdc;
4313859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
4323859Sml29623 
4333859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
4346929Smisaki 	    " ==> nxge_rxdma_cfg_port_default_rdc"));
4353859Sml29623 
4363859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
4376495Sspeer 	actual_rdc = rdc;	/* XXX Hack! */
4383859Sml29623 	rs = npi_rxdma_cfg_default_port_rdc(handle, port, actual_rdc);
4393859Sml29623 
4403859Sml29623 
4413859Sml29623 	if (rs != NPI_SUCCESS) {
4423859Sml29623 		return (NXGE_ERROR | rs);
4433859Sml29623 	}
4443859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
4456929Smisaki 	    " <== nxge_rxdma_cfg_port_default_rdc"));
4463859Sml29623 
4473859Sml29623 	return (NXGE_OK);
4483859Sml29623 }
4493859Sml29623 
4503859Sml29623 nxge_status_t
4513859Sml29623 nxge_rxdma_cfg_rcr_threshold(p_nxge_t nxgep, uint8_t channel,
4523859Sml29623 				    uint16_t pkts)
4533859Sml29623 {
4543859Sml29623 	npi_status_t	rs = NPI_SUCCESS;
4553859Sml29623 	npi_handle_t	handle;
4563859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
4576929Smisaki 	    " ==> nxge_rxdma_cfg_rcr_threshold"));
4583859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
4593859Sml29623 
4603859Sml29623 	rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel, pkts);
4613859Sml29623 
4623859Sml29623 	if (rs != NPI_SUCCESS) {
4633859Sml29623 		return (NXGE_ERROR | rs);
4643859Sml29623 	}
4653859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_threshold"));
4663859Sml29623 	return (NXGE_OK);
4673859Sml29623 }
4683859Sml29623 
4693859Sml29623 nxge_status_t
4703859Sml29623 nxge_rxdma_cfg_rcr_timeout(p_nxge_t nxgep, uint8_t channel,
4713859Sml29623 			    uint16_t tout, uint8_t enable)
4723859Sml29623 {
4733859Sml29623 	npi_status_t	rs = NPI_SUCCESS;
4743859Sml29623 	npi_handle_t	handle;
4753859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " ==> nxge_rxdma_cfg_rcr_timeout"));
4763859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
4773859Sml29623 	if (enable == 0) {
4783859Sml29623 		rs = npi_rxdma_cfg_rdc_rcr_timeout_disable(handle, channel);
4793859Sml29623 	} else {
4803859Sml29623 		rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
4816929Smisaki 		    tout);
4823859Sml29623 	}
4833859Sml29623 
4843859Sml29623 	if (rs != NPI_SUCCESS) {
4853859Sml29623 		return (NXGE_ERROR | rs);
4863859Sml29623 	}
4873859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_timeout"));
4883859Sml29623 	return (NXGE_OK);
4893859Sml29623 }
4903859Sml29623 
4913859Sml29623 nxge_status_t
4923859Sml29623 nxge_enable_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
4933859Sml29623     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p)
4943859Sml29623 {
4953859Sml29623 	npi_handle_t		handle;
4963859Sml29623 	rdc_desc_cfg_t 		rdc_desc;
4973859Sml29623 	p_rcrcfig_b_t		cfgb_p;
4983859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
4993859Sml29623 
5003859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel"));
5013859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
5023859Sml29623 	/*
5033859Sml29623 	 * Use configuration data composed at init time.
5043859Sml29623 	 * Write to hardware the receive ring configurations.
5053859Sml29623 	 */
5063859Sml29623 	rdc_desc.mbox_enable = 1;
5073859Sml29623 	rdc_desc.mbox_addr = mbox_p->mbox_addr;
5083859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
5096929Smisaki 	    "==> nxge_enable_rxdma_channel: mboxp $%p($%p)",
5106929Smisaki 	    mbox_p->mbox_addr, rdc_desc.mbox_addr));
5113859Sml29623 
5123859Sml29623 	rdc_desc.rbr_len = rbr_p->rbb_max;
5133859Sml29623 	rdc_desc.rbr_addr = rbr_p->rbr_addr;
5143859Sml29623 
5153859Sml29623 	switch (nxgep->rx_bksize_code) {
5163859Sml29623 	case RBR_BKSIZE_4K:
5173859Sml29623 		rdc_desc.page_size = SIZE_4KB;
5183859Sml29623 		break;
5193859Sml29623 	case RBR_BKSIZE_8K:
5203859Sml29623 		rdc_desc.page_size = SIZE_8KB;
5213859Sml29623 		break;
5223859Sml29623 	case RBR_BKSIZE_16K:
5233859Sml29623 		rdc_desc.page_size = SIZE_16KB;
5243859Sml29623 		break;
5253859Sml29623 	case RBR_BKSIZE_32K:
5263859Sml29623 		rdc_desc.page_size = SIZE_32KB;
5273859Sml29623 		break;
5283859Sml29623 	}
5293859Sml29623 
5303859Sml29623 	rdc_desc.size0 = rbr_p->npi_pkt_buf_size0;
5313859Sml29623 	rdc_desc.valid0 = 1;
5323859Sml29623 
5333859Sml29623 	rdc_desc.size1 = rbr_p->npi_pkt_buf_size1;
5343859Sml29623 	rdc_desc.valid1 = 1;
5353859Sml29623 
5363859Sml29623 	rdc_desc.size2 = rbr_p->npi_pkt_buf_size2;
5373859Sml29623 	rdc_desc.valid2 = 1;
5383859Sml29623 
5393859Sml29623 	rdc_desc.full_hdr = rcr_p->full_hdr_flag;
5403859Sml29623 	rdc_desc.offset = rcr_p->sw_priv_hdr_len;
5413859Sml29623 
5423859Sml29623 	rdc_desc.rcr_len = rcr_p->comp_size;
5433859Sml29623 	rdc_desc.rcr_addr = rcr_p->rcr_addr;
5443859Sml29623 
5453859Sml29623 	cfgb_p = &(rcr_p->rcr_cfgb);
5463859Sml29623 	rdc_desc.rcr_threshold = cfgb_p->bits.ldw.pthres;
5476495Sspeer 	/* For now, disable this timeout in a guest domain. */
5486495Sspeer 	if (isLDOMguest(nxgep)) {
5496495Sspeer 		rdc_desc.rcr_timeout = 0;
5506495Sspeer 		rdc_desc.rcr_timeout_enable = 0;
5516495Sspeer 	} else {
5526495Sspeer 		rdc_desc.rcr_timeout = cfgb_p->bits.ldw.timeout;
5536495Sspeer 		rdc_desc.rcr_timeout_enable = cfgb_p->bits.ldw.entout;
5546495Sspeer 	}
5553859Sml29623 
5563859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: "
5576929Smisaki 	    "rbr_len qlen %d pagesize code %d rcr_len %d",
5586929Smisaki 	    rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len));
5593859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: "
5606929Smisaki 	    "size 0 %d size 1 %d size 2 %d",
5616929Smisaki 	    rbr_p->npi_pkt_buf_size0, rbr_p->npi_pkt_buf_size1,
5626929Smisaki 	    rbr_p->npi_pkt_buf_size2));
5633859Sml29623 
5643859Sml29623 	rs = npi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc);
5653859Sml29623 	if (rs != NPI_SUCCESS) {
5663859Sml29623 		return (NXGE_ERROR | rs);
5673859Sml29623 	}
5683859Sml29623 
5693859Sml29623 	/*
5703859Sml29623 	 * Enable the timeout and threshold.
5713859Sml29623 	 */
5723859Sml29623 	rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel,
5736929Smisaki 	    rdc_desc.rcr_threshold);
5743859Sml29623 	if (rs != NPI_SUCCESS) {
5753859Sml29623 		return (NXGE_ERROR | rs);
5763859Sml29623 	}
5773859Sml29623 
5783859Sml29623 	rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
5796929Smisaki 	    rdc_desc.rcr_timeout);
5803859Sml29623 	if (rs != NPI_SUCCESS) {
5813859Sml29623 		return (NXGE_ERROR | rs);
5823859Sml29623 	}
5833859Sml29623 
5843859Sml29623 	/* Enable the DMA */
5853859Sml29623 	rs = npi_rxdma_cfg_rdc_enable(handle, channel);
5863859Sml29623 	if (rs != NPI_SUCCESS) {
5873859Sml29623 		return (NXGE_ERROR | rs);
5883859Sml29623 	}
5893859Sml29623 
5903859Sml29623 	/* Kick the DMA engine. */
5913859Sml29623 	npi_rxdma_rdc_rbr_kick(handle, channel, rbr_p->rbb_max);
5923859Sml29623 	/* Clear the rbr empty bit */
5933859Sml29623 	(void) npi_rxdma_channel_rbr_empty_clear(handle, channel);
5943859Sml29623 
5953859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_enable_rxdma_channel"));
5963859Sml29623 
5973859Sml29623 	return (NXGE_OK);
5983859Sml29623 }
5993859Sml29623 
6003859Sml29623 nxge_status_t
6013859Sml29623 nxge_disable_rxdma_channel(p_nxge_t nxgep, uint16_t channel)
6023859Sml29623 {
6033859Sml29623 	npi_handle_t		handle;
6043859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
6053859Sml29623 
6063859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_disable_rxdma_channel"));
6073859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
6083859Sml29623 
6093859Sml29623 	/* disable the DMA */
6103859Sml29623 	rs = npi_rxdma_cfg_rdc_disable(handle, channel);
6113859Sml29623 	if (rs != NPI_SUCCESS) {
6123859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
6136929Smisaki 		    "<== nxge_disable_rxdma_channel:failed (0x%x)",
6146929Smisaki 		    rs));
6153859Sml29623 		return (NXGE_ERROR | rs);
6163859Sml29623 	}
6173859Sml29623 
6183859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_disable_rxdma_channel"));
6193859Sml29623 	return (NXGE_OK);
6203859Sml29623 }
6213859Sml29623 
6223859Sml29623 nxge_status_t
6233859Sml29623 nxge_rxdma_channel_rcrflush(p_nxge_t nxgep, uint8_t channel)
6243859Sml29623 {
6253859Sml29623 	npi_handle_t		handle;
6263859Sml29623 	nxge_status_t		status = NXGE_OK;
6273859Sml29623 
6283859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
6296929Smisaki 	    "<== nxge_init_rxdma_channel_rcrflush"));
6303859Sml29623 
6313859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
6323859Sml29623 	npi_rxdma_rdc_rcr_flush(handle, channel);
6333859Sml29623 
6343859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
6356929Smisaki 	    "<== nxge_init_rxdma_channel_rcrflsh"));
6363859Sml29623 	return (status);
6373859Sml29623 
6383859Sml29623 }
6393859Sml29623 
6403859Sml29623 #define	MID_INDEX(l, r) ((r + l + 1) >> 1)
6413859Sml29623 
6423859Sml29623 #define	TO_LEFT -1
6433859Sml29623 #define	TO_RIGHT 1
6443859Sml29623 #define	BOTH_RIGHT (TO_RIGHT + TO_RIGHT)
6453859Sml29623 #define	BOTH_LEFT (TO_LEFT + TO_LEFT)
6463859Sml29623 #define	IN_MIDDLE (TO_RIGHT + TO_LEFT)
6473859Sml29623 #define	NO_HINT 0xffffffff
6483859Sml29623 
6493859Sml29623 /*ARGSUSED*/
6503859Sml29623 nxge_status_t
6513859Sml29623 nxge_rxbuf_pp_to_vp(p_nxge_t nxgep, p_rx_rbr_ring_t rbr_p,
6523859Sml29623 	uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp,
6533859Sml29623 	uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index)
6543859Sml29623 {
6553859Sml29623 	int			bufsize;
6563859Sml29623 	uint64_t		pktbuf_pp;
6573859Sml29623 	uint64_t 		dvma_addr;
6583859Sml29623 	rxring_info_t 		*ring_info;
6593859Sml29623 	int 			base_side, end_side;
6603859Sml29623 	int 			r_index, l_index, anchor_index;
6613859Sml29623 	int 			found, search_done;
6623859Sml29623 	uint32_t offset, chunk_size, block_size, page_size_mask;
6633859Sml29623 	uint32_t chunk_index, block_index, total_index;
6643859Sml29623 	int 			max_iterations, iteration;
6653859Sml29623 	rxbuf_index_info_t 	*bufinfo;
6663859Sml29623 
6673859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_rxbuf_pp_to_vp"));
6683859Sml29623 
6693859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
6706929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: buf_pp $%p btype %d",
6716929Smisaki 	    pkt_buf_addr_pp,
6726929Smisaki 	    pktbufsz_type));
6735125Sjoycey #if defined(__i386)
6745125Sjoycey 	pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp;
6755125Sjoycey #else
6763859Sml29623 	pktbuf_pp = (uint64_t)pkt_buf_addr_pp;
6775125Sjoycey #endif
6783859Sml29623 
6793859Sml29623 	switch (pktbufsz_type) {
6803859Sml29623 	case 0:
6813859Sml29623 		bufsize = rbr_p->pkt_buf_size0;
6823859Sml29623 		break;
6833859Sml29623 	case 1:
6843859Sml29623 		bufsize = rbr_p->pkt_buf_size1;
6853859Sml29623 		break;
6863859Sml29623 	case 2:
6873859Sml29623 		bufsize = rbr_p->pkt_buf_size2;
6883859Sml29623 		break;
6893859Sml29623 	case RCR_SINGLE_BLOCK:
6903859Sml29623 		bufsize = 0;
6913859Sml29623 		anchor_index = 0;
6923859Sml29623 		break;
6933859Sml29623 	default:
6943859Sml29623 		return (NXGE_ERROR);
6953859Sml29623 	}
6963859Sml29623 
6973859Sml29623 	if (rbr_p->num_blocks == 1) {
6983859Sml29623 		anchor_index = 0;
6993859Sml29623 		ring_info = rbr_p->ring_info;
7003859Sml29623 		bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
7013859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
7026929Smisaki 		    "==> nxge_rxbuf_pp_to_vp: (found, 1 block) "
7036929Smisaki 		    "buf_pp $%p btype %d anchor_index %d "
7046929Smisaki 		    "bufinfo $%p",
7056929Smisaki 		    pkt_buf_addr_pp,
7066929Smisaki 		    pktbufsz_type,
7076929Smisaki 		    anchor_index,
7086929Smisaki 		    bufinfo));
7093859Sml29623 
7103859Sml29623 		goto found_index;
7113859Sml29623 	}
7123859Sml29623 
7133859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
7146929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: "
7156929Smisaki 	    "buf_pp $%p btype %d  anchor_index %d",
7166929Smisaki 	    pkt_buf_addr_pp,
7176929Smisaki 	    pktbufsz_type,
7186929Smisaki 	    anchor_index));
7193859Sml29623 
7203859Sml29623 	ring_info = rbr_p->ring_info;
7213859Sml29623 	found = B_FALSE;
7223859Sml29623 	bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
7233859Sml29623 	iteration = 0;
7243859Sml29623 	max_iterations = ring_info->max_iterations;
7253859Sml29623 		/*
7263859Sml29623 		 * First check if this block has been seen
7273859Sml29623 		 * recently. This is indicated by a hint which
7283859Sml29623 		 * is initialized when the first buffer of the block
7293859Sml29623 		 * is seen. The hint is reset when the last buffer of
7303859Sml29623 		 * the block has been processed.
7313859Sml29623 		 * As three block sizes are supported, three hints
7323859Sml29623 		 * are kept. The idea behind the hints is that once
7333859Sml29623 		 * the hardware  uses a block for a buffer  of that
7343859Sml29623 		 * size, it will use it exclusively for that size
7353859Sml29623 		 * and will use it until it is exhausted. It is assumed
7363859Sml29623 		 * that there would a single block being used for the same
7373859Sml29623 		 * buffer sizes at any given time.
7383859Sml29623 		 */
7393859Sml29623 	if (ring_info->hint[pktbufsz_type] != NO_HINT) {
7403859Sml29623 		anchor_index = ring_info->hint[pktbufsz_type];
7413859Sml29623 		dvma_addr =  bufinfo[anchor_index].dvma_addr;
7423859Sml29623 		chunk_size = bufinfo[anchor_index].buf_size;
7433859Sml29623 		if ((pktbuf_pp >= dvma_addr) &&
7446929Smisaki 		    (pktbuf_pp < (dvma_addr + chunk_size))) {
7453859Sml29623 			found = B_TRUE;
7463859Sml29623 				/*
7473859Sml29623 				 * check if this is the last buffer in the block
7483859Sml29623 				 * If so, then reset the hint for the size;
7493859Sml29623 				 */
7503859Sml29623 
7513859Sml29623 			if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size))
7523859Sml29623 				ring_info->hint[pktbufsz_type] = NO_HINT;
7533859Sml29623 		}
7543859Sml29623 	}
7553859Sml29623 
7563859Sml29623 	if (found == B_FALSE) {
7573859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
7586929Smisaki 		    "==> nxge_rxbuf_pp_to_vp: (!found)"
7596929Smisaki 		    "buf_pp $%p btype %d anchor_index %d",
7606929Smisaki 		    pkt_buf_addr_pp,
7616929Smisaki 		    pktbufsz_type,
7626929Smisaki 		    anchor_index));
7633859Sml29623 
7643859Sml29623 			/*
7653859Sml29623 			 * This is the first buffer of the block of this
7663859Sml29623 			 * size. Need to search the whole information
7673859Sml29623 			 * array.
7683859Sml29623 			 * the search algorithm uses a binary tree search
7693859Sml29623 			 * algorithm. It assumes that the information is
7703859Sml29623 			 * already sorted with increasing order
7713859Sml29623 			 * info[0] < info[1] < info[2]  .... < info[n-1]
7723859Sml29623 			 * where n is the size of the information array
7733859Sml29623 			 */
7743859Sml29623 		r_index = rbr_p->num_blocks - 1;
7753859Sml29623 		l_index = 0;
7763859Sml29623 		search_done = B_FALSE;
7773859Sml29623 		anchor_index = MID_INDEX(r_index, l_index);
7783859Sml29623 		while (search_done == B_FALSE) {
7793859Sml29623 			if ((r_index == l_index) ||
7806929Smisaki 			    (iteration >= max_iterations))
7813859Sml29623 				search_done = B_TRUE;
7823859Sml29623 			end_side = TO_RIGHT; /* to the right */
7833859Sml29623 			base_side = TO_LEFT; /* to the left */
7843859Sml29623 			/* read the DVMA address information and sort it */
7853859Sml29623 			dvma_addr =  bufinfo[anchor_index].dvma_addr;
7863859Sml29623 			chunk_size = bufinfo[anchor_index].buf_size;
7873859Sml29623 			NXGE_DEBUG_MSG((nxgep, RX2_CTL,
7886929Smisaki 			    "==> nxge_rxbuf_pp_to_vp: (searching)"
7896929Smisaki 			    "buf_pp $%p btype %d "
7906929Smisaki 			    "anchor_index %d chunk_size %d dvmaaddr $%p",
7916929Smisaki 			    pkt_buf_addr_pp,
7926929Smisaki 			    pktbufsz_type,
7936929Smisaki 			    anchor_index,
7946929Smisaki 			    chunk_size,
7956929Smisaki 			    dvma_addr));
7963859Sml29623 
7973859Sml29623 			if (pktbuf_pp >= dvma_addr)
7983859Sml29623 				base_side = TO_RIGHT; /* to the right */
7993859Sml29623 			if (pktbuf_pp < (dvma_addr + chunk_size))
8003859Sml29623 				end_side = TO_LEFT; /* to the left */
8013859Sml29623 
8023859Sml29623 			switch (base_side + end_side) {
8036929Smisaki 			case IN_MIDDLE:
8046929Smisaki 				/* found */
8056929Smisaki 				found = B_TRUE;
8066929Smisaki 				search_done = B_TRUE;
8076929Smisaki 				if ((pktbuf_pp + bufsize) <
8086929Smisaki 				    (dvma_addr + chunk_size))
8096929Smisaki 					ring_info->hint[pktbufsz_type] =
8106929Smisaki 					    bufinfo[anchor_index].buf_index;
8116929Smisaki 				break;
8126929Smisaki 			case BOTH_RIGHT:
8136929Smisaki 				/* not found: go to the right */
8146929Smisaki 				l_index = anchor_index + 1;
8156929Smisaki 				anchor_index = MID_INDEX(r_index, l_index);
8166929Smisaki 				break;
8176929Smisaki 
8186929Smisaki 			case BOTH_LEFT:
8196929Smisaki 				/* not found: go to the left */
8206929Smisaki 				r_index = anchor_index - 1;
8216929Smisaki 				anchor_index = MID_INDEX(r_index, l_index);
8226929Smisaki 				break;
8236929Smisaki 			default: /* should not come here */
8246929Smisaki 				return (NXGE_ERROR);
8253859Sml29623 			}
8263859Sml29623 			iteration++;
8273859Sml29623 		}
8283859Sml29623 
8293859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
8306929Smisaki 		    "==> nxge_rxbuf_pp_to_vp: (search done)"
8316929Smisaki 		    "buf_pp $%p btype %d anchor_index %d",
8326929Smisaki 		    pkt_buf_addr_pp,
8336929Smisaki 		    pktbufsz_type,
8346929Smisaki 		    anchor_index));
8353859Sml29623 	}
8363859Sml29623 
8373859Sml29623 	if (found == B_FALSE) {
8383859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
8396929Smisaki 		    "==> nxge_rxbuf_pp_to_vp: (search failed)"
8406929Smisaki 		    "buf_pp $%p btype %d anchor_index %d",
8416929Smisaki 		    pkt_buf_addr_pp,
8426929Smisaki 		    pktbufsz_type,
8436929Smisaki 		    anchor_index));
8443859Sml29623 		return (NXGE_ERROR);
8453859Sml29623 	}
8463859Sml29623 
8473859Sml29623 found_index:
8483859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
8496929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: (FOUND1)"
8506929Smisaki 	    "buf_pp $%p btype %d bufsize %d anchor_index %d",
8516929Smisaki 	    pkt_buf_addr_pp,
8526929Smisaki 	    pktbufsz_type,
8536929Smisaki 	    bufsize,
8546929Smisaki 	    anchor_index));
8553859Sml29623 
8563859Sml29623 	/* index of the first block in this chunk */
8573859Sml29623 	chunk_index = bufinfo[anchor_index].start_index;
8583859Sml29623 	dvma_addr =  bufinfo[anchor_index].dvma_addr;
8593859Sml29623 	page_size_mask = ring_info->block_size_mask;
8603859Sml29623 
8613859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
8626929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: (FOUND3), get chunk)"
8636929Smisaki 	    "buf_pp $%p btype %d bufsize %d "
8646929Smisaki 	    "anchor_index %d chunk_index %d dvma $%p",
8656929Smisaki 	    pkt_buf_addr_pp,
8666929Smisaki 	    pktbufsz_type,
8676929Smisaki 	    bufsize,
8686929Smisaki 	    anchor_index,
8696929Smisaki 	    chunk_index,
8706929Smisaki 	    dvma_addr));
8713859Sml29623 
8723859Sml29623 	offset = pktbuf_pp - dvma_addr; /* offset within the chunk */
8733859Sml29623 	block_size = rbr_p->block_size; /* System  block(page) size */
8743859Sml29623 
8753859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
8766929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: (FOUND4), get chunk)"
8776929Smisaki 	    "buf_pp $%p btype %d bufsize %d "
8786929Smisaki 	    "anchor_index %d chunk_index %d dvma $%p "
8796929Smisaki 	    "offset %d block_size %d",
8806929Smisaki 	    pkt_buf_addr_pp,
8816929Smisaki 	    pktbufsz_type,
8826929Smisaki 	    bufsize,
8836929Smisaki 	    anchor_index,
8846929Smisaki 	    chunk_index,
8856929Smisaki 	    dvma_addr,
8866929Smisaki 	    offset,
8876929Smisaki 	    block_size));
8883859Sml29623 
8893859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> getting total index"));
8903859Sml29623 
8913859Sml29623 	block_index = (offset / block_size); /* index within chunk */
8923859Sml29623 	total_index = chunk_index + block_index;
8933859Sml29623 
8943859Sml29623 
8953859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
8966929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: "
8976929Smisaki 	    "total_index %d dvma_addr $%p "
8986929Smisaki 	    "offset %d block_size %d "
8996929Smisaki 	    "block_index %d ",
9006929Smisaki 	    total_index, dvma_addr,
9016929Smisaki 	    offset, block_size,
9026929Smisaki 	    block_index));
9035125Sjoycey #if defined(__i386)
9045125Sjoycey 	*pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr +
9056929Smisaki 	    (uint32_t)offset);
9065125Sjoycey #else
9075125Sjoycey 	*pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr +
9086929Smisaki 	    (uint64_t)offset);
9095125Sjoycey #endif
9103859Sml29623 
9113859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
9126929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: "
9136929Smisaki 	    "total_index %d dvma_addr $%p "
9146929Smisaki 	    "offset %d block_size %d "
9156929Smisaki 	    "block_index %d "
9166929Smisaki 	    "*pkt_buf_addr_p $%p",
9176929Smisaki 	    total_index, dvma_addr,
9186929Smisaki 	    offset, block_size,
9196929Smisaki 	    block_index,
9206929Smisaki 	    *pkt_buf_addr_p));
9213859Sml29623 
9223859Sml29623 
9233859Sml29623 	*msg_index = total_index;
9243859Sml29623 	*bufoffset =  (offset & page_size_mask);
9253859Sml29623 
9263859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
9276929Smisaki 	    "==> nxge_rxbuf_pp_to_vp: get msg index: "
9286929Smisaki 	    "msg_index %d bufoffset_index %d",
9296929Smisaki 	    *msg_index,
9306929Smisaki 	    *bufoffset));
9313859Sml29623 
9323859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rxbuf_pp_to_vp"));
9333859Sml29623 
9343859Sml29623 	return (NXGE_OK);
9353859Sml29623 }
9363859Sml29623 
9373859Sml29623 /*
9383859Sml29623  * used by quick sort (qsort) function
9393859Sml29623  * to perform comparison
9403859Sml29623  */
9413859Sml29623 static int
9423859Sml29623 nxge_sort_compare(const void *p1, const void *p2)
9433859Sml29623 {
9443859Sml29623 
9453859Sml29623 	rxbuf_index_info_t *a, *b;
9463859Sml29623 
9473859Sml29623 	a = (rxbuf_index_info_t *)p1;
9483859Sml29623 	b = (rxbuf_index_info_t *)p2;
9493859Sml29623 
9503859Sml29623 	if (a->dvma_addr > b->dvma_addr)
9513859Sml29623 		return (1);
9523859Sml29623 	if (a->dvma_addr < b->dvma_addr)
9533859Sml29623 		return (-1);
9543859Sml29623 	return (0);
9553859Sml29623 }
9563859Sml29623 
9573859Sml29623 
9583859Sml29623 
9593859Sml29623 /*
9603859Sml29623  * grabbed this sort implementation from common/syscall/avl.c
9613859Sml29623  *
9623859Sml29623  */
9633859Sml29623 /*
9643859Sml29623  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
9653859Sml29623  * v = Ptr to array/vector of objs
9663859Sml29623  * n = # objs in the array
9673859Sml29623  * s = size of each obj (must be multiples of a word size)
9683859Sml29623  * f = ptr to function to compare two objs
9693859Sml29623  *	returns (-1 = less than, 0 = equal, 1 = greater than
9703859Sml29623  */
9713859Sml29623 void
9723859Sml29623 nxge_ksort(caddr_t v, int n, int s, int (*f)())
9733859Sml29623 {
9743859Sml29623 	int g, i, j, ii;
9753859Sml29623 	unsigned int *p1, *p2;
9763859Sml29623 	unsigned int tmp;
9773859Sml29623 
9783859Sml29623 	/* No work to do */
9793859Sml29623 	if (v == NULL || n <= 1)
9803859Sml29623 		return;
9813859Sml29623 	/* Sanity check on arguments */
9823859Sml29623 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
9833859Sml29623 	ASSERT(s > 0);
9843859Sml29623 
9853859Sml29623 	for (g = n / 2; g > 0; g /= 2) {
9863859Sml29623 		for (i = g; i < n; i++) {
9873859Sml29623 			for (j = i - g; j >= 0 &&
9886929Smisaki 			    (*f)(v + j * s, v + (j + g) * s) == 1;
9896929Smisaki 			    j -= g) {
9903859Sml29623 				p1 = (unsigned *)(v + j * s);
9913859Sml29623 				p2 = (unsigned *)(v + (j + g) * s);
9923859Sml29623 				for (ii = 0; ii < s / 4; ii++) {
9933859Sml29623 					tmp = *p1;
9943859Sml29623 					*p1++ = *p2;
9953859Sml29623 					*p2++ = tmp;
9963859Sml29623 				}
9973859Sml29623 			}
9983859Sml29623 		}
9993859Sml29623 	}
10003859Sml29623 }
10013859Sml29623 
10023859Sml29623 /*
10033859Sml29623  * Initialize data structures required for rxdma
10043859Sml29623  * buffer dvma->vmem address lookup
10053859Sml29623  */
10063859Sml29623 /*ARGSUSED*/
10073859Sml29623 static nxge_status_t
10083859Sml29623 nxge_rxbuf_index_info_init(p_nxge_t nxgep, p_rx_rbr_ring_t rbrp)
10093859Sml29623 {
10103859Sml29623 
10113859Sml29623 	int index;
10123859Sml29623 	rxring_info_t *ring_info;
10133859Sml29623 	int max_iteration = 0, max_index = 0;
10143859Sml29623 
10153859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_rxbuf_index_info_init"));
10163859Sml29623 
10173859Sml29623 	ring_info = rbrp->ring_info;
10183859Sml29623 	ring_info->hint[0] = NO_HINT;
10193859Sml29623 	ring_info->hint[1] = NO_HINT;
10203859Sml29623 	ring_info->hint[2] = NO_HINT;
10213859Sml29623 	max_index = rbrp->num_blocks;
10223859Sml29623 
10233859Sml29623 		/* read the DVMA address information and sort it */
10243859Sml29623 		/* do init of the information array */
10253859Sml29623 
10263859Sml29623 
10273859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
10286929Smisaki 	    " nxge_rxbuf_index_info_init Sort ptrs"));
10293859Sml29623 
10303859Sml29623 		/* sort the array */
10313859Sml29623 	nxge_ksort((void *)ring_info->buffer, max_index,
10326929Smisaki 	    sizeof (rxbuf_index_info_t), nxge_sort_compare);
10333859Sml29623 
10343859Sml29623 
10353859Sml29623 
10363859Sml29623 	for (index = 0; index < max_index; index++) {
10373859Sml29623 		NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
10386929Smisaki 		    " nxge_rxbuf_index_info_init: sorted chunk %d "
10396929Smisaki 		    " ioaddr $%p kaddr $%p size %x",
10406929Smisaki 		    index, ring_info->buffer[index].dvma_addr,
10416929Smisaki 		    ring_info->buffer[index].kaddr,
10426929Smisaki 		    ring_info->buffer[index].buf_size));
10433859Sml29623 	}
10443859Sml29623 
10453859Sml29623 	max_iteration = 0;
10463859Sml29623 	while (max_index >= (1ULL << max_iteration))
10473859Sml29623 		max_iteration++;
10483859Sml29623 	ring_info->max_iterations = max_iteration + 1;
10493859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
10506929Smisaki 	    " nxge_rxbuf_index_info_init Find max iter %d",
10516929Smisaki 	    ring_info->max_iterations));
10523859Sml29623 
10533859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxbuf_index_info_init"));
10543859Sml29623 	return (NXGE_OK);
10553859Sml29623 }
10563859Sml29623 
10573859Sml29623 /* ARGSUSED */
10583859Sml29623 void
10593859Sml29623 nxge_dump_rcr_entry(p_nxge_t nxgep, p_rcr_entry_t entry_p)
10603859Sml29623 {
10613859Sml29623 #ifdef	NXGE_DEBUG
10623859Sml29623 
10633859Sml29623 	uint32_t bptr;
10643859Sml29623 	uint64_t pp;
10653859Sml29623 
10663859Sml29623 	bptr = entry_p->bits.hdw.pkt_buf_addr;
10673859Sml29623 
10683859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
10696929Smisaki 	    "\trcr entry $%p "
10706929Smisaki 	    "\trcr entry 0x%0llx "
10716929Smisaki 	    "\trcr entry 0x%08x "
10726929Smisaki 	    "\trcr entry 0x%08x "
10736929Smisaki 	    "\tvalue 0x%0llx\n"
10746929Smisaki 	    "\tmulti = %d\n"
10756929Smisaki 	    "\tpkt_type = 0x%x\n"
10766929Smisaki 	    "\tzero_copy = %d\n"
10776929Smisaki 	    "\tnoport = %d\n"
10786929Smisaki 	    "\tpromis = %d\n"
10796929Smisaki 	    "\terror = 0x%04x\n"
10806929Smisaki 	    "\tdcf_err = 0x%01x\n"
10816929Smisaki 	    "\tl2_len = %d\n"
10826929Smisaki 	    "\tpktbufsize = %d\n"
10836929Smisaki 	    "\tpkt_buf_addr = $%p\n"
10846929Smisaki 	    "\tpkt_buf_addr (<< 6) = $%p\n",
10856929Smisaki 	    entry_p,
10866929Smisaki 	    *(int64_t *)entry_p,
10876929Smisaki 	    *(int32_t *)entry_p,
10886929Smisaki 	    *(int32_t *)((char *)entry_p + 32),
10896929Smisaki 	    entry_p->value,
10906929Smisaki 	    entry_p->bits.hdw.multi,
10916929Smisaki 	    entry_p->bits.hdw.pkt_type,
10926929Smisaki 	    entry_p->bits.hdw.zero_copy,
10936929Smisaki 	    entry_p->bits.hdw.noport,
10946929Smisaki 	    entry_p->bits.hdw.promis,
10956929Smisaki 	    entry_p->bits.hdw.error,
10966929Smisaki 	    entry_p->bits.hdw.dcf_err,
10976929Smisaki 	    entry_p->bits.hdw.l2_len,
10986929Smisaki 	    entry_p->bits.hdw.pktbufsz,
10996929Smisaki 	    bptr,
11006929Smisaki 	    entry_p->bits.ldw.pkt_buf_addr));
11013859Sml29623 
11023859Sml29623 	pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) <<
11036929Smisaki 	    RCR_PKT_BUF_ADDR_SHIFT;
11043859Sml29623 
11053859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "rcr pp 0x%llx l2 len %d",
11066929Smisaki 	    pp, (*(int64_t *)entry_p >> 40) & 0x3fff));
11073859Sml29623 #endif
11083859Sml29623 }
11093859Sml29623 
11103859Sml29623 void
11113859Sml29623 nxge_rxdma_regs_dump(p_nxge_t nxgep, int rdc)
11123859Sml29623 {
11133859Sml29623 	npi_handle_t		handle;
11143859Sml29623 	rbr_stat_t 		rbr_stat;
11153859Sml29623 	addr44_t 		hd_addr;
11163859Sml29623 	addr44_t 		tail_addr;
11173859Sml29623 	uint16_t 		qlen;
11183859Sml29623 
11193859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
11206929Smisaki 	    "==> nxge_rxdma_regs_dump: rdc channel %d", rdc));
11213859Sml29623 
11223859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
11233859Sml29623 
11243859Sml29623 	/* RBR head */
11253859Sml29623 	hd_addr.addr = 0;
11263859Sml29623 	(void) npi_rxdma_rdc_rbr_head_get(handle, rdc, &hd_addr);
11275165Syc148097 #if defined(__i386)
11283859Sml29623 	printf("nxge_rxdma_regs_dump: got hdptr $%p \n",
11296929Smisaki 	    (void *)(uint32_t)hd_addr.addr);
11305125Sjoycey #else
11315165Syc148097 	printf("nxge_rxdma_regs_dump: got hdptr $%p \n",
11326929Smisaki 	    (void *)hd_addr.addr);
11335125Sjoycey #endif
11343859Sml29623 
11353859Sml29623 	/* RBR stats */
11363859Sml29623 	(void) npi_rxdma_rdc_rbr_stat_get(handle, rdc, &rbr_stat);
11373859Sml29623 	printf("nxge_rxdma_regs_dump: rbr len %d \n", rbr_stat.bits.ldw.qlen);
11383859Sml29623 
11393859Sml29623 	/* RCR tail */
11403859Sml29623 	tail_addr.addr = 0;
11413859Sml29623 	(void) npi_rxdma_rdc_rcr_tail_get(handle, rdc, &tail_addr);
11425165Syc148097 #if defined(__i386)
11433859Sml29623 	printf("nxge_rxdma_regs_dump: got tail ptr $%p \n",
11446929Smisaki 	    (void *)(uint32_t)tail_addr.addr);
11455125Sjoycey #else
11465165Syc148097 	printf("nxge_rxdma_regs_dump: got tail ptr $%p \n",
11476929Smisaki 	    (void *)tail_addr.addr);
11485125Sjoycey #endif
11493859Sml29623 
11503859Sml29623 	/* RCR qlen */
11513859Sml29623 	(void) npi_rxdma_rdc_rcr_qlen_get(handle, rdc, &qlen);
11523859Sml29623 	printf("nxge_rxdma_regs_dump: rcr len %x \n", qlen);
11533859Sml29623 
11543859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
11556929Smisaki 	    "<== nxge_rxdma_regs_dump: rdc rdc %d", rdc));
11563859Sml29623 }
11573859Sml29623 
11583859Sml29623 void
11593859Sml29623 nxge_rxdma_stop(p_nxge_t nxgep)
11603859Sml29623 {
11613859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop"));
11623859Sml29623 
11633859Sml29623 	(void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP);
11643859Sml29623 	(void) nxge_rx_mac_disable(nxgep);
11653859Sml29623 	(void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_STOP);
11663859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop"));
11673859Sml29623 }
11683859Sml29623 
11693859Sml29623 void
11703859Sml29623 nxge_rxdma_stop_reinit(p_nxge_t nxgep)
11713859Sml29623 {
11723859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_reinit"));
11733859Sml29623 
11743859Sml29623 	(void) nxge_rxdma_stop(nxgep);
11753859Sml29623 	(void) nxge_uninit_rxdma_channels(nxgep);
11763859Sml29623 	(void) nxge_init_rxdma_channels(nxgep);
11773859Sml29623 
11783859Sml29623 #ifndef	AXIS_DEBUG_LB
11793859Sml29623 	(void) nxge_xcvr_init(nxgep);
11803859Sml29623 	(void) nxge_link_monitor(nxgep, LINK_MONITOR_START);
11813859Sml29623 #endif
11823859Sml29623 	(void) nxge_rx_mac_enable(nxgep);
11833859Sml29623 
11843859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_reinit"));
11853859Sml29623 }
11863859Sml29623 
11873859Sml29623 nxge_status_t
11883859Sml29623 nxge_rxdma_hw_mode(p_nxge_t nxgep, boolean_t enable)
11893859Sml29623 {
11906495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
11916495Sspeer 	nxge_status_t status;
11926495Sspeer 	npi_status_t rs;
11936495Sspeer 	int rdc;
11943859Sml29623 
11953859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
11966929Smisaki 	    "==> nxge_rxdma_hw_mode: mode %d", enable));
11973859Sml29623 
11983859Sml29623 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
11993859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
12006495Sspeer 		    "<== nxge_rxdma_mode: not initialized"));
12013859Sml29623 		return (NXGE_ERROR);
12023859Sml29623 	}
12036495Sspeer 
12046495Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
12056495Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
12066495Sspeer 		    "<== nxge_tx_port_fatal_err_recover: "
12076495Sspeer 		    "NULL ring pointer(s)"));
12083859Sml29623 		return (NXGE_ERROR);
12093859Sml29623 	}
12103859Sml29623 
12116495Sspeer 	if (set->owned.map == 0) {
12126495Sspeer 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
12136495Sspeer 		    "nxge_rxdma_regs_dump_channels: no channels"));
12146495Sspeer 		return (NULL);
12156495Sspeer 	}
12166495Sspeer 
12176495Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
12186495Sspeer 		if ((1 << rdc) & set->owned.map) {
12196495Sspeer 			rx_rbr_ring_t *ring =
12206495Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
12216495Sspeer 			npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep);
12226495Sspeer 			if (ring) {
12236495Sspeer 				if (enable) {
12246495Sspeer 					NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
12256495Sspeer 					    "==> nxge_rxdma_hw_mode: "
12266495Sspeer 					    "channel %d (enable)", rdc));
12276495Sspeer 					rs = npi_rxdma_cfg_rdc_enable
12286495Sspeer 					    (handle, rdc);
12296495Sspeer 				} else {
12306495Sspeer 					NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
12316495Sspeer 					    "==> nxge_rxdma_hw_mode: "
12326495Sspeer 					    "channel %d disable)", rdc));
12336495Sspeer 					rs = npi_rxdma_cfg_rdc_disable
12346495Sspeer 					    (handle, rdc);
12356495Sspeer 				}
12366495Sspeer 			}
12373859Sml29623 		}
12383859Sml29623 	}
12393859Sml29623 
12403859Sml29623 	status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
12413859Sml29623 
12423859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
12436929Smisaki 	    "<== nxge_rxdma_hw_mode: status 0x%x", status));
12443859Sml29623 
12453859Sml29623 	return (status);
12463859Sml29623 }
12473859Sml29623 
12483859Sml29623 void
12493859Sml29623 nxge_rxdma_enable_channel(p_nxge_t nxgep, uint16_t channel)
12503859Sml29623 {
12513859Sml29623 	npi_handle_t		handle;
12523859Sml29623 
12533859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
12546929Smisaki 	    "==> nxge_rxdma_enable_channel: channel %d", channel));
12553859Sml29623 
12563859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
12573859Sml29623 	(void) npi_rxdma_cfg_rdc_enable(handle, channel);
12583859Sml29623 
12593859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_enable_channel"));
12603859Sml29623 }
12613859Sml29623 
12623859Sml29623 void
12633859Sml29623 nxge_rxdma_disable_channel(p_nxge_t nxgep, uint16_t channel)
12643859Sml29623 {
12653859Sml29623 	npi_handle_t		handle;
12663859Sml29623 
12673859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
12686929Smisaki 	    "==> nxge_rxdma_disable_channel: channel %d", channel));
12693859Sml29623 
12703859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
12713859Sml29623 	(void) npi_rxdma_cfg_rdc_disable(handle, channel);
12723859Sml29623 
12733859Sml29623 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_disable_channel"));
12743859Sml29623 }
12753859Sml29623 
12763859Sml29623 void
12773859Sml29623 nxge_hw_start_rx(p_nxge_t nxgep)
12783859Sml29623 {
12793859Sml29623 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_hw_start_rx"));
12803859Sml29623 
12813859Sml29623 	(void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_START);
12823859Sml29623 	(void) nxge_rx_mac_enable(nxgep);
12833859Sml29623 
12843859Sml29623 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_hw_start_rx"));
12853859Sml29623 }
12863859Sml29623 
12873859Sml29623 /*ARGSUSED*/
12883859Sml29623 void
12893859Sml29623 nxge_fixup_rxdma_rings(p_nxge_t nxgep)
12903859Sml29623 {
12916495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
12926495Sspeer 	int rdc;
12933859Sml29623 
12943859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_fixup_rxdma_rings"));
12953859Sml29623 
12966495Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
12976495Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
12986495Sspeer 		    "<== nxge_tx_port_fatal_err_recover: "
12996495Sspeer 		    "NULL ring pointer(s)"));
13003859Sml29623 		return;
13013859Sml29623 	}
13023859Sml29623 
13036495Sspeer 	if (set->owned.map == 0) {
13043859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
13056495Sspeer 		    "nxge_rxdma_regs_dump_channels: no channels"));
13063859Sml29623 		return;
13073859Sml29623 	}
13086495Sspeer 
13096495Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
13106495Sspeer 		if ((1 << rdc) & set->owned.map) {
13116495Sspeer 			rx_rbr_ring_t *ring =
13126495Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
13136495Sspeer 			if (ring) {
13146495Sspeer 				nxge_rxdma_hw_stop(nxgep, rdc);
13156495Sspeer 				NXGE_DEBUG_MSG((nxgep, RX_CTL,
13166929Smisaki 				    "==> nxge_fixup_rxdma_rings: "
13176929Smisaki 				    "channel %d ring $%px",
13186929Smisaki 				    rdc, ring));
13196495Sspeer 				(void) nxge_rxdma_fixup_channel
13206495Sspeer 				    (nxgep, rdc, rdc);
13216495Sspeer 			}
13226495Sspeer 		}
13233859Sml29623 	}
13243859Sml29623 
13253859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_fixup_rxdma_rings"));
13263859Sml29623 }
13273859Sml29623 
13283859Sml29623 void
13293859Sml29623 nxge_rxdma_fix_channel(p_nxge_t nxgep, uint16_t channel)
13303859Sml29623 {
13313859Sml29623 	int		i;
13323859Sml29623 
13333859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fix_channel"));
13343859Sml29623 	i = nxge_rxdma_get_ring_index(nxgep, channel);
13353859Sml29623 	if (i < 0) {
13363859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
13376929Smisaki 		    "<== nxge_rxdma_fix_channel: no entry found"));
13383859Sml29623 		return;
13393859Sml29623 	}
13403859Sml29623 
13413859Sml29623 	nxge_rxdma_fixup_channel(nxgep, channel, i);
13423859Sml29623 
13436495Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fix_channel"));
13443859Sml29623 }
13453859Sml29623 
13463859Sml29623 void
13473859Sml29623 nxge_rxdma_fixup_channel(p_nxge_t nxgep, uint16_t channel, int entry)
13483859Sml29623 {
13493859Sml29623 	int			ndmas;
13503859Sml29623 	p_rx_rbr_rings_t 	rx_rbr_rings;
13513859Sml29623 	p_rx_rbr_ring_t		*rbr_rings;
13523859Sml29623 	p_rx_rcr_rings_t 	rx_rcr_rings;
13533859Sml29623 	p_rx_rcr_ring_t		*rcr_rings;
13543859Sml29623 	p_rx_mbox_areas_t 	rx_mbox_areas_p;
13553859Sml29623 	p_rx_mbox_t		*rx_mbox_p;
13563859Sml29623 	p_nxge_dma_pool_t	dma_buf_poolp;
13573859Sml29623 	p_nxge_dma_pool_t	dma_cntl_poolp;
13583859Sml29623 	p_rx_rbr_ring_t 	rbrp;
13593859Sml29623 	p_rx_rcr_ring_t 	rcrp;
13603859Sml29623 	p_rx_mbox_t 		mboxp;
13613859Sml29623 	p_nxge_dma_common_t 	dmap;
13623859Sml29623 	nxge_status_t		status = NXGE_OK;
13633859Sml29623 
13643859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fixup_channel"));
13653859Sml29623 
13663859Sml29623 	(void) nxge_rxdma_stop_channel(nxgep, channel);
13673859Sml29623 
13683859Sml29623 	dma_buf_poolp = nxgep->rx_buf_pool_p;
13693859Sml29623 	dma_cntl_poolp = nxgep->rx_cntl_pool_p;
13703859Sml29623 
13713859Sml29623 	if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) {
13723859Sml29623 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
13736929Smisaki 		    "<== nxge_rxdma_fixup_channel: buf not allocated"));
13743859Sml29623 		return;
13753859Sml29623 	}
13763859Sml29623 
13773859Sml29623 	ndmas = dma_buf_poolp->ndmas;
13783859Sml29623 	if (!ndmas) {
13793859Sml29623 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
13806929Smisaki 		    "<== nxge_rxdma_fixup_channel: no dma allocated"));
13813859Sml29623 		return;
13823859Sml29623 	}
13833859Sml29623 
13843859Sml29623 	rx_rbr_rings = nxgep->rx_rbr_rings;
13853859Sml29623 	rx_rcr_rings = nxgep->rx_rcr_rings;
13863859Sml29623 	rbr_rings = rx_rbr_rings->rbr_rings;
13873859Sml29623 	rcr_rings = rx_rcr_rings->rcr_rings;
13883859Sml29623 	rx_mbox_areas_p = nxgep->rx_mbox_areas_p;
13893859Sml29623 	rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
13903859Sml29623 
13913859Sml29623 	/* Reinitialize the receive block and completion rings */
13923859Sml29623 	rbrp = (p_rx_rbr_ring_t)rbr_rings[entry],
13936929Smisaki 	    rcrp = (p_rx_rcr_ring_t)rcr_rings[entry],
13946929Smisaki 	    mboxp = (p_rx_mbox_t)rx_mbox_p[entry];
13953859Sml29623 
13963859Sml29623 
13973859Sml29623 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
13983859Sml29623 	rbrp->rbr_rd_index = 0;
13993859Sml29623 	rcrp->comp_rd_index = 0;
14003859Sml29623 	rcrp->comp_wt_index = 0;
14013859Sml29623 
14023859Sml29623 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
14033859Sml29623 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
14043859Sml29623 
14053859Sml29623 	status = nxge_rxdma_start_channel(nxgep, channel,
14066929Smisaki 	    rbrp, rcrp, mboxp);
14073859Sml29623 	if (status != NXGE_OK) {
14083859Sml29623 		goto nxge_rxdma_fixup_channel_fail;
14093859Sml29623 	}
14103859Sml29623 	if (status != NXGE_OK) {
14113859Sml29623 		goto nxge_rxdma_fixup_channel_fail;
14123859Sml29623 	}
14133859Sml29623 
14143859Sml29623 nxge_rxdma_fixup_channel_fail:
14153859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
14166929Smisaki 	    "==> nxge_rxdma_fixup_channel: failed (0x%08x)", status));
14173859Sml29623 
14183859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fixup_channel"));
14193859Sml29623 }
14203859Sml29623 
14216495Sspeer /* ARGSUSED */
14223859Sml29623 int
14233859Sml29623 nxge_rxdma_get_ring_index(p_nxge_t nxgep, uint16_t channel)
14243859Sml29623 {
14256495Sspeer 	return (channel);
14263859Sml29623 }
14273859Sml29623 
14283859Sml29623 p_rx_rbr_ring_t
14293859Sml29623 nxge_rxdma_get_rbr_ring(p_nxge_t nxgep, uint16_t channel)
14303859Sml29623 {
14316495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
14326495Sspeer 	nxge_channel_t rdc;
14333859Sml29623 
14343859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
14356929Smisaki 	    "==> nxge_rxdma_get_rbr_ring: channel %d", channel));
14363859Sml29623 
14376495Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
14386495Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
14396495Sspeer 		    "<== nxge_rxdma_get_rbr_ring: "
14406495Sspeer 		    "NULL ring pointer(s)"));
14413859Sml29623 		return (NULL);
14423859Sml29623 	}
14436495Sspeer 
14446495Sspeer 	if (set->owned.map == 0) {
14453859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
14466495Sspeer 		    "<== nxge_rxdma_get_rbr_ring: no channels"));
14473859Sml29623 		return (NULL);
14483859Sml29623 	}
14493859Sml29623 
14506495Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
14516495Sspeer 		if ((1 << rdc) & set->owned.map) {
14526495Sspeer 			rx_rbr_ring_t *ring =
14536495Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
14546495Sspeer 			if (ring) {
14556495Sspeer 				if (channel == ring->rdc) {
14566495Sspeer 					NXGE_DEBUG_MSG((nxgep, RX_CTL,
14576495Sspeer 					    "==> nxge_rxdma_get_rbr_ring: "
14586495Sspeer 					    "channel %d ring $%p", rdc, ring));
14596495Sspeer 					return (ring);
14606495Sspeer 				}
14616495Sspeer 			}
14623859Sml29623 		}
14633859Sml29623 	}
14643859Sml29623 
14653859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
14666929Smisaki 	    "<== nxge_rxdma_get_rbr_ring: not found"));
14673859Sml29623 
14683859Sml29623 	return (NULL);
14693859Sml29623 }
14703859Sml29623 
14713859Sml29623 p_rx_rcr_ring_t
14723859Sml29623 nxge_rxdma_get_rcr_ring(p_nxge_t nxgep, uint16_t channel)
14733859Sml29623 {
14746495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
14756495Sspeer 	nxge_channel_t rdc;
14763859Sml29623 
14773859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
14786929Smisaki 	    "==> nxge_rxdma_get_rcr_ring: channel %d", channel));
14793859Sml29623 
14806495Sspeer 	if (nxgep->rx_rcr_rings == 0 || nxgep->rx_rcr_rings->rcr_rings == 0) {
14816495Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
14826495Sspeer 		    "<== nxge_rxdma_get_rcr_ring: "
14836495Sspeer 		    "NULL ring pointer(s)"));
14843859Sml29623 		return (NULL);
14853859Sml29623 	}
14866495Sspeer 
14876495Sspeer 	if (set->owned.map == 0) {
14883859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
14896495Sspeer 		    "<== nxge_rxdma_get_rbr_ring: no channels"));
14903859Sml29623 		return (NULL);
14913859Sml29623 	}
14923859Sml29623 
14936495Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
14946495Sspeer 		if ((1 << rdc) & set->owned.map) {
14956495Sspeer 			rx_rcr_ring_t *ring =
14966495Sspeer 			    nxgep->rx_rcr_rings->rcr_rings[rdc];
14976495Sspeer 			if (ring) {
14986495Sspeer 				if (channel == ring->rdc) {
14996495Sspeer 					NXGE_DEBUG_MSG((nxgep, RX_CTL,
15006495Sspeer 					    "==> nxge_rxdma_get_rcr_ring: "
15016495Sspeer 					    "channel %d ring $%p", rdc, ring));
15026495Sspeer 					return (ring);
15036495Sspeer 				}
15046495Sspeer 			}
15053859Sml29623 		}
15063859Sml29623 	}
15073859Sml29623 
15083859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
15096929Smisaki 	    "<== nxge_rxdma_get_rcr_ring: not found"));
15103859Sml29623 
15113859Sml29623 	return (NULL);
15123859Sml29623 }
15133859Sml29623 
15143859Sml29623 /*
15153859Sml29623  * Static functions start here.
15163859Sml29623  */
15173859Sml29623 static p_rx_msg_t
15183859Sml29623 nxge_allocb(size_t size, uint32_t pri, p_nxge_dma_common_t dmabuf_p)
15193859Sml29623 {
15203859Sml29623 	p_rx_msg_t nxge_mp 		= NULL;
15213859Sml29623 	p_nxge_dma_common_t		dmamsg_p;
15223859Sml29623 	uchar_t 			*buffer;
15233859Sml29623 
15243859Sml29623 	nxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP);
15253859Sml29623 	if (nxge_mp == NULL) {
15264185Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
15276929Smisaki 		    "Allocation of a rx msg failed."));
15283859Sml29623 		goto nxge_allocb_exit;
15293859Sml29623 	}
15303859Sml29623 
15313859Sml29623 	nxge_mp->use_buf_pool = B_FALSE;
15323859Sml29623 	if (dmabuf_p) {
15333859Sml29623 		nxge_mp->use_buf_pool = B_TRUE;
15343859Sml29623 		dmamsg_p = (p_nxge_dma_common_t)&nxge_mp->buf_dma;
15353859Sml29623 		*dmamsg_p = *dmabuf_p;
15363859Sml29623 		dmamsg_p->nblocks = 1;
15373859Sml29623 		dmamsg_p->block_size = size;
15383859Sml29623 		dmamsg_p->alength = size;
15393859Sml29623 		buffer = (uchar_t *)dmabuf_p->kaddrp;
15403859Sml29623 
15413859Sml29623 		dmabuf_p->kaddrp = (void *)
15426929Smisaki 		    ((char *)dmabuf_p->kaddrp + size);
15433859Sml29623 		dmabuf_p->ioaddr_pp = (void *)
15446929Smisaki 		    ((char *)dmabuf_p->ioaddr_pp + size);
15453859Sml29623 		dmabuf_p->alength -= size;
15463859Sml29623 		dmabuf_p->offset += size;
15473859Sml29623 		dmabuf_p->dma_cookie.dmac_laddress += size;
15483859Sml29623 		dmabuf_p->dma_cookie.dmac_size -= size;
15493859Sml29623 
15503859Sml29623 	} else {
15513859Sml29623 		buffer = KMEM_ALLOC(size, KM_NOSLEEP);
15523859Sml29623 		if (buffer == NULL) {
15534185Sspeer 			NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
15546929Smisaki 			    "Allocation of a receive page failed."));
15553859Sml29623 			goto nxge_allocb_fail1;
15563859Sml29623 		}
15573859Sml29623 	}
15583859Sml29623 
15593859Sml29623 	nxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &nxge_mp->freeb);
15603859Sml29623 	if (nxge_mp->rx_mblk_p == NULL) {
15614185Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "desballoc failed."));
15623859Sml29623 		goto nxge_allocb_fail2;
15633859Sml29623 	}
15643859Sml29623 
15653859Sml29623 	nxge_mp->buffer = buffer;
15663859Sml29623 	nxge_mp->block_size = size;
15673859Sml29623 	nxge_mp->freeb.free_func = (void (*)())nxge_freeb;
15683859Sml29623 	nxge_mp->freeb.free_arg = (caddr_t)nxge_mp;
15693859Sml29623 	nxge_mp->ref_cnt = 1;
15703859Sml29623 	nxge_mp->free = B_TRUE;
15713859Sml29623 	nxge_mp->rx_use_bcopy = B_FALSE;
15723859Sml29623 
15733859Sml29623 	atomic_inc_32(&nxge_mblks_pending);
15743859Sml29623 
15753859Sml29623 	goto nxge_allocb_exit;
15763859Sml29623 
15773859Sml29623 nxge_allocb_fail2:
15783859Sml29623 	if (!nxge_mp->use_buf_pool) {
15793859Sml29623 		KMEM_FREE(buffer, size);
15803859Sml29623 	}
15813859Sml29623 
15823859Sml29623 nxge_allocb_fail1:
15833859Sml29623 	KMEM_FREE(nxge_mp, sizeof (rx_msg_t));
15843859Sml29623 	nxge_mp = NULL;
15853859Sml29623 
15863859Sml29623 nxge_allocb_exit:
15873859Sml29623 	return (nxge_mp);
15883859Sml29623 }
15893859Sml29623 
15903859Sml29623 p_mblk_t
15913859Sml29623 nxge_dupb(p_rx_msg_t nxge_mp, uint_t offset, size_t size)
15923859Sml29623 {
15933859Sml29623 	p_mblk_t mp;
15943859Sml29623 
15953859Sml29623 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "==> nxge_dupb"));
15963859Sml29623 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "nxge_mp = $%p "
15976929Smisaki 	    "offset = 0x%08X "
15986929Smisaki 	    "size = 0x%08X",
15996929Smisaki 	    nxge_mp, offset, size));
16003859Sml29623 
16013859Sml29623 	mp = desballoc(&nxge_mp->buffer[offset], size,
16026929Smisaki 	    0, &nxge_mp->freeb);
16033859Sml29623 	if (mp == NULL) {
16043859Sml29623 		NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
16053859Sml29623 		goto nxge_dupb_exit;
16063859Sml29623 	}
16073859Sml29623 	atomic_inc_32(&nxge_mp->ref_cnt);
16083859Sml29623 
16093859Sml29623 
16103859Sml29623 nxge_dupb_exit:
16113859Sml29623 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p",
16126929Smisaki 	    nxge_mp));
16133859Sml29623 	return (mp);
16143859Sml29623 }
16153859Sml29623 
16163859Sml29623 p_mblk_t
16173859Sml29623 nxge_dupb_bcopy(p_rx_msg_t nxge_mp, uint_t offset, size_t size)
16183859Sml29623 {
16193859Sml29623 	p_mblk_t mp;
16203859Sml29623 	uchar_t *dp;
16213859Sml29623 
16223859Sml29623 	mp = allocb(size + NXGE_RXBUF_EXTRA, 0);
16233859Sml29623 	if (mp == NULL) {
16243859Sml29623 		NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
16253859Sml29623 		goto nxge_dupb_bcopy_exit;
16263859Sml29623 	}
16273859Sml29623 	dp = mp->b_rptr = mp->b_rptr + NXGE_RXBUF_EXTRA;
16283859Sml29623 	bcopy((void *)&nxge_mp->buffer[offset], dp, size);
16293859Sml29623 	mp->b_wptr = dp + size;
16303859Sml29623 
16313859Sml29623 nxge_dupb_bcopy_exit:
16323859Sml29623 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p",
16336929Smisaki 	    nxge_mp));
16343859Sml29623 	return (mp);
16353859Sml29623 }
16363859Sml29623 
16373859Sml29623 void nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p,
16383859Sml29623 	p_rx_msg_t rx_msg_p);
16393859Sml29623 
16403859Sml29623 void
16413859Sml29623 nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p)
16423859Sml29623 {
16433859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_post_page"));
16443859Sml29623 
16453859Sml29623 	/* Reuse this buffer */
16463859Sml29623 	rx_msg_p->free = B_FALSE;
16473859Sml29623 	rx_msg_p->cur_usage_cnt = 0;
16483859Sml29623 	rx_msg_p->max_usage_cnt = 0;
16493859Sml29623 	rx_msg_p->pkt_buf_size = 0;
16503859Sml29623 
16513859Sml29623 	if (rx_rbr_p->rbr_use_bcopy) {
16523859Sml29623 		rx_msg_p->rx_use_bcopy = B_FALSE;
16533859Sml29623 		atomic_dec_32(&rx_rbr_p->rbr_consumed);
16543859Sml29623 	}
16553859Sml29623 
16563859Sml29623 	/*
16573859Sml29623 	 * Get the rbr header pointer and its offset index.
16583859Sml29623 	 */
16593859Sml29623 	MUTEX_ENTER(&rx_rbr_p->post_lock);
16603859Sml29623 	rx_rbr_p->rbr_wr_index =  ((rx_rbr_p->rbr_wr_index + 1) &
16616929Smisaki 	    rx_rbr_p->rbr_wrap_mask);
16623859Sml29623 	rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr;
16633859Sml29623 	MUTEX_EXIT(&rx_rbr_p->post_lock);
16645770Sml29623 	npi_rxdma_rdc_rbr_kick(NXGE_DEV_NPI_HANDLE(nxgep),
16655770Sml29623 	    rx_rbr_p->rdc, 1);
16663859Sml29623 
16673859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
16686929Smisaki 	    "<== nxge_post_page (channel %d post_next_index %d)",
16696929Smisaki 	    rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index));
16703859Sml29623 
16713859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_post_page"));
16723859Sml29623 }
16733859Sml29623 
16743859Sml29623 void
16753859Sml29623 nxge_freeb(p_rx_msg_t rx_msg_p)
16763859Sml29623 {
16773859Sml29623 	size_t size;
16783859Sml29623 	uchar_t *buffer = NULL;
16793859Sml29623 	int ref_cnt;
16804874Sml29623 	boolean_t free_state = B_FALSE;
16813859Sml29623 
16825170Stm144005 	rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p;
16835170Stm144005 
16843859Sml29623 	NXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> nxge_freeb"));
16853859Sml29623 	NXGE_DEBUG_MSG((NULL, MEM2_CTL,
16866929Smisaki 	    "nxge_freeb:rx_msg_p = $%p (block pending %d)",
16876929Smisaki 	    rx_msg_p, nxge_mblks_pending));
16883859Sml29623 
16894874Sml29623 	/*
16904874Sml29623 	 * First we need to get the free state, then
16914874Sml29623 	 * atomic decrement the reference count to prevent
16924874Sml29623 	 * the race condition with the interrupt thread that
16934874Sml29623 	 * is processing a loaned up buffer block.
16944874Sml29623 	 */
16954874Sml29623 	free_state = rx_msg_p->free;
16963859Sml29623 	ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1);
16973859Sml29623 	if (!ref_cnt) {
16985770Sml29623 		atomic_dec_32(&nxge_mblks_pending);
16993859Sml29623 		buffer = rx_msg_p->buffer;
17003859Sml29623 		size = rx_msg_p->block_size;
17013859Sml29623 		NXGE_DEBUG_MSG((NULL, MEM2_CTL, "nxge_freeb: "
17026929Smisaki 		    "will free: rx_msg_p = $%p (block pending %d)",
17036929Smisaki 		    rx_msg_p, nxge_mblks_pending));
17043859Sml29623 
17053859Sml29623 		if (!rx_msg_p->use_buf_pool) {
17063859Sml29623 			KMEM_FREE(buffer, size);
17073859Sml29623 		}
17083859Sml29623 
17093859Sml29623 		KMEM_FREE(rx_msg_p, sizeof (rx_msg_t));
17105170Stm144005 
17115759Smisaki 		if (ring) {
17125759Smisaki 			/*
17135759Smisaki 			 * Decrement the receive buffer ring's reference
17145759Smisaki 			 * count, too.
17155759Smisaki 			 */
17165759Smisaki 			atomic_dec_32(&ring->rbr_ref_cnt);
17175759Smisaki 
17185759Smisaki 			/*
17196495Sspeer 			 * Free the receive buffer ring, if
17205759Smisaki 			 * 1. all the receive buffers have been freed
17215759Smisaki 			 * 2. and we are in the proper state (that is,
17225759Smisaki 			 *    we are not UNMAPPING).
17235759Smisaki 			 */
17245759Smisaki 			if (ring->rbr_ref_cnt == 0 &&
17255759Smisaki 			    ring->rbr_state == RBR_UNMAPPED) {
17266495Sspeer 				/*
17276495Sspeer 				 * Free receive data buffers,
17286495Sspeer 				 * buffer index information
17296495Sspeer 				 * (rxring_info) and
17306495Sspeer 				 * the message block ring.
17316495Sspeer 				 */
17326495Sspeer 				NXGE_DEBUG_MSG((NULL, RX_CTL,
17336495Sspeer 				    "nxge_freeb:rx_msg_p = $%p "
17346495Sspeer 				    "(block pending %d) free buffers",
17356495Sspeer 				    rx_msg_p, nxge_mblks_pending));
17366495Sspeer 				nxge_rxdma_databuf_free(ring);
17376495Sspeer 				if (ring->ring_info) {
17386495Sspeer 					KMEM_FREE(ring->ring_info,
17396495Sspeer 					    sizeof (rxring_info_t));
17406495Sspeer 				}
17416495Sspeer 
17426495Sspeer 				if (ring->rx_msg_ring) {
17436495Sspeer 					KMEM_FREE(ring->rx_msg_ring,
17446495Sspeer 					    ring->tnblocks *
17456495Sspeer 					    sizeof (p_rx_msg_t));
17466495Sspeer 				}
17475759Smisaki 				KMEM_FREE(ring, sizeof (*ring));
17485759Smisaki 			}
17495170Stm144005 		}
17503859Sml29623 		return;
17513859Sml29623 	}
17523859Sml29623 
17533859Sml29623 	/*
17543859Sml29623 	 * Repost buffer.
17553859Sml29623 	 */
17565759Smisaki 	if (free_state && (ref_cnt == 1) && ring) {
17573859Sml29623 		NXGE_DEBUG_MSG((NULL, RX_CTL,
17583859Sml29623 		    "nxge_freeb: post page $%p:", rx_msg_p));
17595170Stm144005 		if (ring->rbr_state == RBR_POSTING)
17605170Stm144005 			nxge_post_page(rx_msg_p->nxgep, ring, rx_msg_p);
17613859Sml29623 	}
17623859Sml29623 
17633859Sml29623 	NXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== nxge_freeb"));
17643859Sml29623 }
17653859Sml29623 
17663859Sml29623 uint_t
17673859Sml29623 nxge_rx_intr(void *arg1, void *arg2)
17683859Sml29623 {
17693859Sml29623 	p_nxge_ldv_t		ldvp = (p_nxge_ldv_t)arg1;
17703859Sml29623 	p_nxge_t		nxgep = (p_nxge_t)arg2;
17713859Sml29623 	p_nxge_ldg_t		ldgp;
17723859Sml29623 	uint8_t			channel;
17733859Sml29623 	npi_handle_t		handle;
17743859Sml29623 	rx_dma_ctl_stat_t	cs;
17753859Sml29623 
17763859Sml29623 #ifdef	NXGE_DEBUG
17773859Sml29623 	rxdma_cfig1_t		cfg;
17783859Sml29623 #endif
17793859Sml29623 	uint_t 			serviced = DDI_INTR_UNCLAIMED;
17803859Sml29623 
17813859Sml29623 	if (ldvp == NULL) {
17823859Sml29623 		NXGE_DEBUG_MSG((NULL, INT_CTL,
17836929Smisaki 		    "<== nxge_rx_intr: arg2 $%p arg1 $%p",
17846929Smisaki 		    nxgep, ldvp));
17853859Sml29623 
17863859Sml29623 		return (DDI_INTR_CLAIMED);
17873859Sml29623 	}
17883859Sml29623 
17893859Sml29623 	if (arg2 == NULL || (void *)ldvp->nxgep != arg2) {
17903859Sml29623 		nxgep = ldvp->nxgep;
17913859Sml29623 	}
17926602Sspeer 
17936602Sspeer 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
17946602Sspeer 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
17956602Sspeer 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
17966602Sspeer 		    "<== nxge_rx_intr: interface not started or intialized"));
17976602Sspeer 		return (DDI_INTR_CLAIMED);
17986602Sspeer 	}
17996602Sspeer 
18003859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
18016929Smisaki 	    "==> nxge_rx_intr: arg2 $%p arg1 $%p",
18026929Smisaki 	    nxgep, ldvp));
18033859Sml29623 
18043859Sml29623 	/*
18053859Sml29623 	 * This interrupt handler is for a specific
18063859Sml29623 	 * receive dma channel.
18073859Sml29623 	 */
18083859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
18093859Sml29623 	/*
18103859Sml29623 	 * Get the control and status for this channel.
18113859Sml29623 	 */
18123859Sml29623 	channel = ldvp->channel;
18133859Sml29623 	ldgp = ldvp->ldgp;
18143859Sml29623 	RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, channel, &cs.value);
18153859Sml29623 
18163859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_intr:channel %d "
18176929Smisaki 	    "cs 0x%016llx rcrto 0x%x rcrthres %x",
18186929Smisaki 	    channel,
18196929Smisaki 	    cs.value,
18206929Smisaki 	    cs.bits.hdw.rcrto,
18216929Smisaki 	    cs.bits.hdw.rcrthres));
18223859Sml29623 
18236495Sspeer 	nxge_rx_pkts_vring(nxgep, ldvp->vdma_index, cs);
18243859Sml29623 	serviced = DDI_INTR_CLAIMED;
18253859Sml29623 
18263859Sml29623 	/* error events. */
18273859Sml29623 	if (cs.value & RX_DMA_CTL_STAT_ERROR) {
18286495Sspeer 		(void) nxge_rx_err_evnts(nxgep, channel, cs);
18293859Sml29623 	}
18303859Sml29623 
18313859Sml29623 nxge_intr_exit:
18323859Sml29623 	/*
18333859Sml29623 	 * Enable the mailbox update interrupt if we want
18343859Sml29623 	 * to use mailbox. We probably don't need to use
18353859Sml29623 	 * mailbox as it only saves us one pio read.
18363859Sml29623 	 * Also write 1 to rcrthres and rcrto to clear
18373859Sml29623 	 * these two edge triggered bits.
18383859Sml29623 	 */
18393859Sml29623 
18403859Sml29623 	cs.value &= RX_DMA_CTL_STAT_WR1C;
18413859Sml29623 	cs.bits.hdw.mex = 1;
18423859Sml29623 	RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel,
18436929Smisaki 	    cs.value);
18443859Sml29623 
18453859Sml29623 	/*
18463859Sml29623 	 * Rearm this logical group if this is a single device
18473859Sml29623 	 * group.
18483859Sml29623 	 */
18493859Sml29623 	if (ldgp->nldvs == 1) {
18503859Sml29623 		ldgimgm_t		mgm;
18513859Sml29623 		mgm.value = 0;
18523859Sml29623 		mgm.bits.ldw.arm = 1;
18533859Sml29623 		mgm.bits.ldw.timer = ldgp->ldg_timer;
18546495Sspeer 		if (isLDOMguest(nxgep)) {
18556495Sspeer 			nxge_hio_ldgimgn(nxgep, ldgp);
18566495Sspeer 		} else {
18576495Sspeer 			NXGE_REG_WR64(handle,
18583859Sml29623 			    LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg),
18593859Sml29623 			    mgm.value);
18606495Sspeer 		}
18613859Sml29623 	}
18623859Sml29623 
18633859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_intr: serviced %d",
18646929Smisaki 	    serviced));
18653859Sml29623 	return (serviced);
18663859Sml29623 }
18673859Sml29623 
18683859Sml29623 /*
18693859Sml29623  * Process the packets received in the specified logical device
18703859Sml29623  * and pass up a chain of message blocks to the upper layer.
18713859Sml29623  */
18723859Sml29623 static void
18736495Sspeer nxge_rx_pkts_vring(p_nxge_t nxgep, uint_t vindex, rx_dma_ctl_stat_t cs)
18743859Sml29623 {
18753859Sml29623 	p_mblk_t		mp;
18763859Sml29623 	p_rx_rcr_ring_t		rcrp;
18773859Sml29623 
18783859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts_vring"));
18796495Sspeer 	rcrp = nxgep->rx_rcr_rings->rcr_rings[vindex];
18806495Sspeer 	if (rcrp->poll_flag) {
18816495Sspeer 		/* It is in the poll mode */
18826495Sspeer 		return;
18836495Sspeer 	}
18846495Sspeer 
18856495Sspeer 	if ((mp = nxge_rx_pkts(nxgep, rcrp, cs, -1)) == NULL) {
18863859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
18876929Smisaki 		    "<== nxge_rx_pkts_vring: no mp"));
18883859Sml29623 		return;
18893859Sml29623 	}
18903859Sml29623 
18913859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts_vring: $%p",
18926929Smisaki 	    mp));
18933859Sml29623 
18943859Sml29623 #ifdef  NXGE_DEBUG
18953859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
18966929Smisaki 		    "==> nxge_rx_pkts_vring:calling mac_rx "
18976929Smisaki 		    "LEN %d mp $%p mp->b_cont $%p mp->b_next $%p rcrp $%p "
18986929Smisaki 		    "mac_handle $%p",
18996929Smisaki 		    mp->b_wptr - mp->b_rptr,
19006929Smisaki 		    mp, mp->b_cont, mp->b_next,
19016929Smisaki 		    rcrp, rcrp->rcr_mac_handle));
19023859Sml29623 
19033859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
19046929Smisaki 		    "==> nxge_rx_pkts_vring: dump packets "
19056929Smisaki 		    "(mp $%p b_rptr $%p b_wptr $%p):\n %s",
19066929Smisaki 		    mp,
19076929Smisaki 		    mp->b_rptr,
19086929Smisaki 		    mp->b_wptr,
19096929Smisaki 		    nxge_dump_packet((char *)mp->b_rptr,
19106929Smisaki 		    mp->b_wptr - mp->b_rptr)));
19113859Sml29623 		if (mp->b_cont) {
19123859Sml29623 			NXGE_DEBUG_MSG((nxgep, RX_CTL,
19136929Smisaki 			    "==> nxge_rx_pkts_vring: dump b_cont packets "
19146929Smisaki 			    "(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s",
19156929Smisaki 			    mp->b_cont,
19166929Smisaki 			    mp->b_cont->b_rptr,
19176929Smisaki 			    mp->b_cont->b_wptr,
19186929Smisaki 			    nxge_dump_packet((char *)mp->b_cont->b_rptr,
19196929Smisaki 			    mp->b_cont->b_wptr - mp->b_cont->b_rptr)));
19203859Sml29623 		}
19213859Sml29623 		if (mp->b_next) {
19223859Sml29623 			NXGE_DEBUG_MSG((nxgep, RX_CTL,
19236929Smisaki 			    "==> nxge_rx_pkts_vring: dump next packets "
19246929Smisaki 			    "(b_rptr $%p): %s",
19256929Smisaki 			    mp->b_next->b_rptr,
19266929Smisaki 			    nxge_dump_packet((char *)mp->b_next->b_rptr,
19276929Smisaki 			    mp->b_next->b_wptr - mp->b_next->b_rptr)));
19283859Sml29623 		}
19293859Sml29623 #endif
19303859Sml29623 
19316495Sspeer 	if (!isLDOMguest(nxgep))
19326495Sspeer 		mac_rx(nxgep->mach, rcrp->rcr_mac_handle, mp);
19336495Sspeer #if defined(sun4v)
19346495Sspeer 	else {			/* isLDOMguest(nxgep) */
19356495Sspeer 		nxge_hio_data_t *nhd = (nxge_hio_data_t *)
19366495Sspeer 		    nxgep->nxge_hw_p->hio;
19376495Sspeer 		nx_vio_fp_t *vio = &nhd->hio.vio;
19386495Sspeer 
19396495Sspeer 		if (vio->cb.vio_net_rx_cb) {
19406495Sspeer 			(*vio->cb.vio_net_rx_cb)
19416495Sspeer 			    (nxgep->hio_vr->vhp, mp);
19426495Sspeer 		}
19436495Sspeer 	}
19446495Sspeer #endif
19453859Sml29623 }
19463859Sml29623 
19473859Sml29623 
19483859Sml29623 /*
19493859Sml29623  * This routine is the main packet receive processing function.
19503859Sml29623  * It gets the packet type, error code, and buffer related
19513859Sml29623  * information from the receive completion entry.
19523859Sml29623  * How many completion entries to process is based on the number of packets
19533859Sml29623  * queued by the hardware, a hardware maintained tail pointer
19543859Sml29623  * and a configurable receive packet count.
19553859Sml29623  *
19563859Sml29623  * A chain of message blocks will be created as result of processing
19573859Sml29623  * the completion entries. This chain of message blocks will be returned and
19583859Sml29623  * a hardware control status register will be updated with the number of
19593859Sml29623  * packets were removed from the hardware queue.
19603859Sml29623  *
19613859Sml29623  */
19626495Sspeer static mblk_t *
19636495Sspeer nxge_rx_pkts(p_nxge_t nxgep, p_rx_rcr_ring_t rcr_p, rx_dma_ctl_stat_t cs,
19646495Sspeer     int bytes_to_pickup)
19653859Sml29623 {
19663859Sml29623 	npi_handle_t		handle;
19673859Sml29623 	uint8_t			channel;
19683859Sml29623 	uint32_t		comp_rd_index;
19693859Sml29623 	p_rcr_entry_t		rcr_desc_rd_head_p;
19703859Sml29623 	p_rcr_entry_t		rcr_desc_rd_head_pp;
19713859Sml29623 	p_mblk_t		nmp, mp_cont, head_mp, *tail_mp;
19723859Sml29623 	uint16_t		qlen, nrcr_read, npkt_read;
19736495Sspeer 	uint32_t		qlen_hw;
19743859Sml29623 	boolean_t		multi;
19756495Sspeer 	rcrcfig_b_t		rcr_cfg_b;
19766495Sspeer 	int			totallen = 0;
19773859Sml29623 #if defined(_BIG_ENDIAN)
19783859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
19793859Sml29623 #endif
19803859Sml29623 
19816495Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: "
19826929Smisaki 	    "channel %d", rcr_p->rdc));
19833859Sml29623 
19843859Sml29623 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
19853859Sml29623 		return (NULL);
19863859Sml29623 	}
19873859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
19883859Sml29623 	channel = rcr_p->rdc;
19893859Sml29623 
19903859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
19916929Smisaki 	    "==> nxge_rx_pkts: START: rcr channel %d "
19926929Smisaki 	    "head_p $%p head_pp $%p  index %d ",
19936929Smisaki 	    channel, rcr_p->rcr_desc_rd_head_p,
19946929Smisaki 	    rcr_p->rcr_desc_rd_head_pp,
19956929Smisaki 	    rcr_p->comp_rd_index));
19963859Sml29623 
19973859Sml29623 
19983859Sml29623 #if !defined(_BIG_ENDIAN)
19993859Sml29623 	qlen = RXDMA_REG_READ32(handle, RCRSTAT_A_REG, channel) & 0xffff;
20003859Sml29623 #else
20013859Sml29623 	rs = npi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen);
20023859Sml29623 	if (rs != NPI_SUCCESS) {
20036495Sspeer 		NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: "
20043859Sml29623 		"channel %d, get qlen failed 0x%08x",
20056929Smisaki 		    channel, rs));
20063859Sml29623 		return (NULL);
20073859Sml29623 	}
20083859Sml29623 #endif
20093859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts:rcr channel %d "
20106929Smisaki 	    "qlen %d", channel, qlen));
20113859Sml29623 
20123859Sml29623 
20133859Sml29623 
20143859Sml29623 	if (!qlen) {
20153859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
20166929Smisaki 		    "==> nxge_rx_pkts:rcr channel %d "
20176929Smisaki 		    "qlen %d (no pkts)", channel, qlen));
20183859Sml29623 
20193859Sml29623 		return (NULL);
20203859Sml29623 	}
20213859Sml29623 
20223859Sml29623 	comp_rd_index = rcr_p->comp_rd_index;
20233859Sml29623 
20243859Sml29623 	rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p;
20253859Sml29623 	rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp;
20263859Sml29623 	nrcr_read = npkt_read = 0;
20273859Sml29623 
20283859Sml29623 	/*
20293859Sml29623 	 * Number of packets queued
20303859Sml29623 	 * (The jumbo or multi packet will be counted as only one
20313859Sml29623 	 *  packets and it may take up more than one completion entry).
20323859Sml29623 	 */
20333859Sml29623 	qlen_hw = (qlen < nxge_max_rx_pkts) ?
20346929Smisaki 	    qlen : nxge_max_rx_pkts;
20353859Sml29623 	head_mp = NULL;
20363859Sml29623 	tail_mp = &head_mp;
20373859Sml29623 	nmp = mp_cont = NULL;
20383859Sml29623 	multi = B_FALSE;
20393859Sml29623 
20403859Sml29623 	while (qlen_hw) {
20413859Sml29623 
20423859Sml29623 #ifdef NXGE_DEBUG
20433859Sml29623 		nxge_dump_rcr_entry(nxgep, rcr_desc_rd_head_p);
20443859Sml29623 #endif
20453859Sml29623 		/*
20463859Sml29623 		 * Process one completion ring entry.
20473859Sml29623 		 */
20483859Sml29623 		nxge_receive_packet(nxgep,
20496929Smisaki 		    rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont);
20503859Sml29623 
20513859Sml29623 		/*
20523859Sml29623 		 * message chaining modes
20533859Sml29623 		 */
20543859Sml29623 		if (nmp) {
20553859Sml29623 			nmp->b_next = NULL;
20563859Sml29623 			if (!multi && !mp_cont) { /* frame fits a partition */
20573859Sml29623 				*tail_mp = nmp;
20583859Sml29623 				tail_mp = &nmp->b_next;
20596495Sspeer 				totallen += MBLKL(nmp);
20603859Sml29623 				nmp = NULL;
20613859Sml29623 			} else if (multi && !mp_cont) { /* first segment */
20623859Sml29623 				*tail_mp = nmp;
20633859Sml29623 				tail_mp = &nmp->b_cont;
20646495Sspeer 				totallen += MBLKL(nmp);
20653859Sml29623 			} else if (multi && mp_cont) {	/* mid of multi segs */
20663859Sml29623 				*tail_mp = mp_cont;
20673859Sml29623 				tail_mp = &mp_cont->b_cont;
20686495Sspeer 				totallen += MBLKL(mp_cont);
20693859Sml29623 			} else if (!multi && mp_cont) { /* last segment */
20703859Sml29623 				*tail_mp = mp_cont;
20713859Sml29623 				tail_mp = &nmp->b_next;
20726495Sspeer 				totallen += MBLKL(mp_cont);
20733859Sml29623 				nmp = NULL;
20743859Sml29623 			}
20753859Sml29623 		}
20763859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
20776929Smisaki 		    "==> nxge_rx_pkts: loop: rcr channel %d "
20786929Smisaki 		    "before updating: multi %d "
20796929Smisaki 		    "nrcr_read %d "
20806929Smisaki 		    "npk read %d "
20816929Smisaki 		    "head_pp $%p  index %d ",
20826929Smisaki 		    channel,
20836929Smisaki 		    multi,
20846929Smisaki 		    nrcr_read, npkt_read, rcr_desc_rd_head_pp,
20856929Smisaki 		    comp_rd_index));
20863859Sml29623 
20873859Sml29623 		if (!multi) {
20883859Sml29623 			qlen_hw--;
20893859Sml29623 			npkt_read++;
20903859Sml29623 		}
20913859Sml29623 
20923859Sml29623 		/*
20933859Sml29623 		 * Update the next read entry.
20943859Sml29623 		 */
20953859Sml29623 		comp_rd_index = NEXT_ENTRY(comp_rd_index,
20966929Smisaki 		    rcr_p->comp_wrap_mask);
20973859Sml29623 
20983859Sml29623 		rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p,
20996929Smisaki 		    rcr_p->rcr_desc_first_p,
21006929Smisaki 		    rcr_p->rcr_desc_last_p);
21013859Sml29623 
21023859Sml29623 		nrcr_read++;
21033859Sml29623 
21043859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
21056929Smisaki 		    "<== nxge_rx_pkts: (SAM, process one packet) "
21066929Smisaki 		    "nrcr_read %d",
21076929Smisaki 		    nrcr_read));
21083859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
21096929Smisaki 		    "==> nxge_rx_pkts: loop: rcr channel %d "
21106929Smisaki 		    "multi %d "
21116929Smisaki 		    "nrcr_read %d "
21126929Smisaki 		    "npk read %d "
21136929Smisaki 		    "head_pp $%p  index %d ",
21146929Smisaki 		    channel,
21156929Smisaki 		    multi,
21166929Smisaki 		    nrcr_read, npkt_read, rcr_desc_rd_head_pp,
21176929Smisaki 		    comp_rd_index));
21183859Sml29623 
21196495Sspeer 		if ((bytes_to_pickup != -1) &&
21206495Sspeer 		    (totallen >= bytes_to_pickup)) {
21216495Sspeer 			break;
21226495Sspeer 		}
21233859Sml29623 	}
21243859Sml29623 
21253859Sml29623 	rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp;
21263859Sml29623 	rcr_p->comp_rd_index = comp_rd_index;
21273859Sml29623 	rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p;
21283859Sml29623 
21293859Sml29623 	if ((nxgep->intr_timeout != rcr_p->intr_timeout) ||
21306929Smisaki 	    (nxgep->intr_threshold != rcr_p->intr_threshold)) {
21313859Sml29623 		rcr_p->intr_timeout = nxgep->intr_timeout;
21323859Sml29623 		rcr_p->intr_threshold = nxgep->intr_threshold;
21333859Sml29623 		rcr_cfg_b.value = 0x0ULL;
21343859Sml29623 		if (rcr_p->intr_timeout)
21353859Sml29623 			rcr_cfg_b.bits.ldw.entout = 1;
21363859Sml29623 		rcr_cfg_b.bits.ldw.timeout = rcr_p->intr_timeout;
21373859Sml29623 		rcr_cfg_b.bits.ldw.pthres = rcr_p->intr_threshold;
21383859Sml29623 		RXDMA_REG_WRITE64(handle, RCRCFIG_B_REG,
21396929Smisaki 		    channel, rcr_cfg_b.value);
21403859Sml29623 	}
21413859Sml29623 
21423859Sml29623 	cs.bits.ldw.pktread = npkt_read;
21433859Sml29623 	cs.bits.ldw.ptrread = nrcr_read;
21443859Sml29623 	RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG,
21456929Smisaki 	    channel, cs.value);
21463859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
21476929Smisaki 	    "==> nxge_rx_pkts: EXIT: rcr channel %d "
21486929Smisaki 	    "head_pp $%p  index %016llx ",
21496929Smisaki 	    channel,
21506929Smisaki 	    rcr_p->rcr_desc_rd_head_pp,
21516929Smisaki 	    rcr_p->comp_rd_index));
21523859Sml29623 	/*
21533859Sml29623 	 * Update RCR buffer pointer read and number of packets
21543859Sml29623 	 * read.
21553859Sml29623 	 */
21563859Sml29623 
21573859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_pkts"));
21583859Sml29623 	return (head_mp);
21593859Sml29623 }
21603859Sml29623 
21613859Sml29623 void
21623859Sml29623 nxge_receive_packet(p_nxge_t nxgep,
21633859Sml29623     p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p,
21643859Sml29623     boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont)
21653859Sml29623 {
21663859Sml29623 	p_mblk_t		nmp = NULL;
21673859Sml29623 	uint64_t		multi;
21683859Sml29623 	uint64_t		dcf_err;
21693859Sml29623 	uint8_t			channel;
21703859Sml29623 
21713859Sml29623 	boolean_t		first_entry = B_TRUE;
21723859Sml29623 	boolean_t		is_tcp_udp = B_FALSE;
21733859Sml29623 	boolean_t		buffer_free = B_FALSE;
21743859Sml29623 	boolean_t		error_send_up = B_FALSE;
21753859Sml29623 	uint8_t			error_type;
21763859Sml29623 	uint16_t		l2_len;
21773859Sml29623 	uint16_t		skip_len;
21783859Sml29623 	uint8_t			pktbufsz_type;
21793859Sml29623 	uint64_t		rcr_entry;
21803859Sml29623 	uint64_t		*pkt_buf_addr_pp;
21813859Sml29623 	uint64_t		*pkt_buf_addr_p;
21823859Sml29623 	uint32_t		buf_offset;
21833859Sml29623 	uint32_t		bsize;
21843859Sml29623 	uint32_t		error_disp_cnt;
21853859Sml29623 	uint32_t		msg_index;
21863859Sml29623 	p_rx_rbr_ring_t		rx_rbr_p;
21873859Sml29623 	p_rx_msg_t 		*rx_msg_ring_p;
21883859Sml29623 	p_rx_msg_t		rx_msg_p;
21893859Sml29623 	uint16_t		sw_offset_bytes = 0, hdr_size = 0;
21903859Sml29623 	nxge_status_t		status = NXGE_OK;
21913859Sml29623 	boolean_t		is_valid = B_FALSE;
21923859Sml29623 	p_nxge_rx_ring_stats_t	rdc_stats;
21933859Sml29623 	uint32_t		bytes_read;
21943859Sml29623 	uint64_t		pkt_type;
21953859Sml29623 	uint64_t		frag;
21966028Ssbehera 	boolean_t		pkt_too_long_err = B_FALSE;
21973859Sml29623 #ifdef	NXGE_DEBUG
21983859Sml29623 	int			dump_len;
21993859Sml29623 #endif
22003859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_receive_packet"));
22013859Sml29623 	first_entry = (*mp == NULL) ? B_TRUE : B_FALSE;
22023859Sml29623 
22033859Sml29623 	rcr_entry = *((uint64_t *)rcr_desc_rd_head_p);
22043859Sml29623 
22053859Sml29623 	multi = (rcr_entry & RCR_MULTI_MASK);
22063859Sml29623 	dcf_err = (rcr_entry & RCR_DCF_ERROR_MASK);
22073859Sml29623 	pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK);
22083859Sml29623 
22093859Sml29623 	error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT);
22103859Sml29623 	frag = (rcr_entry & RCR_FRAG_MASK);
22113859Sml29623 
22123859Sml29623 	l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT);
22133859Sml29623 
22143859Sml29623 	pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >>
22156929Smisaki 	    RCR_PKTBUFSZ_SHIFT);
22165125Sjoycey #if defined(__i386)
22175125Sjoycey 	pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry &
22186929Smisaki 	    RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT);
22195125Sjoycey #else
22203859Sml29623 	pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) <<
22216929Smisaki 	    RCR_PKT_BUF_ADDR_SHIFT);
22225125Sjoycey #endif
22233859Sml29623 
22243859Sml29623 	channel = rcr_p->rdc;
22253859Sml29623 
22263859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
22276929Smisaki 	    "==> nxge_receive_packet: entryp $%p entry 0x%0llx "
22286929Smisaki 	    "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx "
22296929Smisaki 	    "error_type 0x%x pkt_type 0x%x  "
22306929Smisaki 	    "pktbufsz_type %d ",
22316929Smisaki 	    rcr_desc_rd_head_p,
22326929Smisaki 	    rcr_entry, pkt_buf_addr_pp, l2_len,
22336929Smisaki 	    multi,
22346929Smisaki 	    error_type,
22356929Smisaki 	    pkt_type,
22366929Smisaki 	    pktbufsz_type));
22373859Sml29623 
22383859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
22396929Smisaki 	    "==> nxge_receive_packet: entryp $%p entry 0x%0llx "
22406929Smisaki 	    "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx "
22416929Smisaki 	    "error_type 0x%x pkt_type 0x%x ", rcr_desc_rd_head_p,
22426929Smisaki 	    rcr_entry, pkt_buf_addr_pp, l2_len,
22436929Smisaki 	    multi,
22446929Smisaki 	    error_type,
22456929Smisaki 	    pkt_type));
22463859Sml29623 
22473859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
22486929Smisaki 	    "==> (rbr) nxge_receive_packet: entry 0x%0llx "
22496929Smisaki 	    "full pkt_buf_addr_pp $%p l2_len %d",
22506929Smisaki 	    rcr_entry, pkt_buf_addr_pp, l2_len));
22513859Sml29623 
22523859Sml29623 	/* get the stats ptr */
22533859Sml29623 	rdc_stats = rcr_p->rdc_stats;
22543859Sml29623 
22553859Sml29623 	if (!l2_len) {
22563859Sml29623 
22573859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
22586929Smisaki 		    "<== nxge_receive_packet: failed: l2 length is 0."));
22593859Sml29623 		return;
22603859Sml29623 	}
22613859Sml29623 
22626028Ssbehera 	/*
22636028Ssbehera 	 * Sofware workaround for BMAC hardware limitation that allows
22646028Ssbehera 	 * maxframe size of 1526, instead of 1522 for non-jumbo and 0x2406
22656028Ssbehera 	 * instead of 0x2400 for jumbo.
22666028Ssbehera 	 */
22676028Ssbehera 	if (l2_len > nxgep->mac.maxframesize) {
22686028Ssbehera 		pkt_too_long_err = B_TRUE;
22696028Ssbehera 	}
22706028Ssbehera 
22714185Sspeer 	/* Hardware sends us 4 bytes of CRC as no stripping is done.  */
22724185Sspeer 	l2_len -= ETHERFCSL;
22734185Sspeer 
22743859Sml29623 	/* shift 6 bits to get the full io address */
22755125Sjoycey #if defined(__i386)
22765125Sjoycey 	pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp <<
22776929Smisaki 	    RCR_PKT_BUF_ADDR_SHIFT_FULL);
22785125Sjoycey #else
22793859Sml29623 	pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp <<
22806929Smisaki 	    RCR_PKT_BUF_ADDR_SHIFT_FULL);
22815125Sjoycey #endif
22823859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
22836929Smisaki 	    "==> (rbr) nxge_receive_packet: entry 0x%0llx "
22846929Smisaki 	    "full pkt_buf_addr_pp $%p l2_len %d",
22856929Smisaki 	    rcr_entry, pkt_buf_addr_pp, l2_len));
22863859Sml29623 
22873859Sml29623 	rx_rbr_p = rcr_p->rx_rbr_p;
22883859Sml29623 	rx_msg_ring_p = rx_rbr_p->rx_msg_ring;
22893859Sml29623 
22903859Sml29623 	if (first_entry) {
22913859Sml29623 		hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL :
22926929Smisaki 		    RXDMA_HDR_SIZE_DEFAULT);
22933859Sml29623 
22943859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
22956929Smisaki 		    "==> nxge_receive_packet: first entry 0x%016llx "
22966929Smisaki 		    "pkt_buf_addr_pp $%p l2_len %d hdr %d",
22976929Smisaki 		    rcr_entry, pkt_buf_addr_pp, l2_len,
22986929Smisaki 		    hdr_size));
22993859Sml29623 	}
23003859Sml29623 
23013859Sml29623 	MUTEX_ENTER(&rcr_p->lock);
23023859Sml29623 	MUTEX_ENTER(&rx_rbr_p->lock);
23033859Sml29623 
23043859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
23056929Smisaki 	    "==> (rbr 1) nxge_receive_packet: entry 0x%0llx "
23066929Smisaki 	    "full pkt_buf_addr_pp $%p l2_len %d",
23076929Smisaki 	    rcr_entry, pkt_buf_addr_pp, l2_len));
23083859Sml29623 
23093859Sml29623 	/*
23103859Sml29623 	 * Packet buffer address in the completion entry points
23113859Sml29623 	 * to the starting buffer address (offset 0).
23123859Sml29623 	 * Use the starting buffer address to locate the corresponding
23133859Sml29623 	 * kernel address.
23143859Sml29623 	 */
23153859Sml29623 	status = nxge_rxbuf_pp_to_vp(nxgep, rx_rbr_p,
23166929Smisaki 	    pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p,
23176929Smisaki 	    &buf_offset,
23186929Smisaki 	    &msg_index);
23193859Sml29623 
23203859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
23216929Smisaki 	    "==> (rbr 2) nxge_receive_packet: entry 0x%0llx "
23226929Smisaki 	    "full pkt_buf_addr_pp $%p l2_len %d",
23236929Smisaki 	    rcr_entry, pkt_buf_addr_pp, l2_len));
23243859Sml29623 
23253859Sml29623 	if (status != NXGE_OK) {
23263859Sml29623 		MUTEX_EXIT(&rx_rbr_p->lock);
23273859Sml29623 		MUTEX_EXIT(&rcr_p->lock);
23283859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
23296929Smisaki 		    "<== nxge_receive_packet: found vaddr failed %d",
23306929Smisaki 		    status));
23313859Sml29623 		return;
23323859Sml29623 	}
23333859Sml29623 
23343859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
23356929Smisaki 	    "==> (rbr 3) nxge_receive_packet: entry 0x%0llx "
23366929Smisaki 	    "full pkt_buf_addr_pp $%p l2_len %d",
23376929Smisaki 	    rcr_entry, pkt_buf_addr_pp, l2_len));
23383859Sml29623 
23393859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
23406929Smisaki 	    "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx "
23416929Smisaki 	    "full pkt_buf_addr_pp $%p l2_len %d",
23426929Smisaki 	    msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
23433859Sml29623 
23443859Sml29623 	rx_msg_p = rx_msg_ring_p[msg_index];
23453859Sml29623 
23463859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
23476929Smisaki 	    "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx "
23486929Smisaki 	    "full pkt_buf_addr_pp $%p l2_len %d",
23496929Smisaki 	    msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
23503859Sml29623 
23513859Sml29623 	switch (pktbufsz_type) {
23523859Sml29623 	case RCR_PKTBUFSZ_0:
23533859Sml29623 		bsize = rx_rbr_p->pkt_buf_size0_bytes;
23543859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
23556929Smisaki 		    "==> nxge_receive_packet: 0 buf %d", bsize));
23563859Sml29623 		break;
23573859Sml29623 	case RCR_PKTBUFSZ_1:
23583859Sml29623 		bsize = rx_rbr_p->pkt_buf_size1_bytes;
23593859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
23606929Smisaki 		    "==> nxge_receive_packet: 1 buf %d", bsize));
23613859Sml29623 		break;
23623859Sml29623 	case RCR_PKTBUFSZ_2:
23633859Sml29623 		bsize = rx_rbr_p->pkt_buf_size2_bytes;
23643859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
23656929Smisaki 		    "==> nxge_receive_packet: 2 buf %d", bsize));
23663859Sml29623 		break;
23673859Sml29623 	case RCR_SINGLE_BLOCK:
23683859Sml29623 		bsize = rx_msg_p->block_size;
23693859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
23706929Smisaki 		    "==> nxge_receive_packet: single %d", bsize));
23713859Sml29623 
23723859Sml29623 		break;
23733859Sml29623 	default:
23743859Sml29623 		MUTEX_EXIT(&rx_rbr_p->lock);
23753859Sml29623 		MUTEX_EXIT(&rcr_p->lock);
23763859Sml29623 		return;
23773859Sml29623 	}
23783859Sml29623 
23793859Sml29623 	DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma,
23806929Smisaki 	    (buf_offset + sw_offset_bytes),
23816929Smisaki 	    (hdr_size + l2_len),
23826929Smisaki 	    DDI_DMA_SYNC_FORCPU);
23833859Sml29623 
23843859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
23856929Smisaki 	    "==> nxge_receive_packet: after first dump:usage count"));
23863859Sml29623 
23873859Sml29623 	if (rx_msg_p->cur_usage_cnt == 0) {
23883859Sml29623 		if (rx_rbr_p->rbr_use_bcopy) {
23893859Sml29623 			atomic_inc_32(&rx_rbr_p->rbr_consumed);
23903859Sml29623 			if (rx_rbr_p->rbr_consumed <
23916929Smisaki 			    rx_rbr_p->rbr_threshold_hi) {
23923859Sml29623 				if (rx_rbr_p->rbr_threshold_lo == 0 ||
23936929Smisaki 				    ((rx_rbr_p->rbr_consumed >=
23946929Smisaki 				    rx_rbr_p->rbr_threshold_lo) &&
23956929Smisaki 				    (rx_rbr_p->rbr_bufsize_type >=
23966929Smisaki 				    pktbufsz_type))) {
23973859Sml29623 					rx_msg_p->rx_use_bcopy = B_TRUE;
23983859Sml29623 				}
23993859Sml29623 			} else {
24003859Sml29623 				rx_msg_p->rx_use_bcopy = B_TRUE;
24013859Sml29623 			}
24023859Sml29623 		}
24033859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
24046929Smisaki 		    "==> nxge_receive_packet: buf %d (new block) ",
24056929Smisaki 		    bsize));
24063859Sml29623 
24073859Sml29623 		rx_msg_p->pkt_buf_size_code = pktbufsz_type;
24083859Sml29623 		rx_msg_p->pkt_buf_size = bsize;
24093859Sml29623 		rx_msg_p->cur_usage_cnt = 1;
24103859Sml29623 		if (pktbufsz_type == RCR_SINGLE_BLOCK) {
24113859Sml29623 			NXGE_DEBUG_MSG((nxgep, RX2_CTL,
24126929Smisaki 			    "==> nxge_receive_packet: buf %d "
24136929Smisaki 			    "(single block) ",
24146929Smisaki 			    bsize));
24153859Sml29623 			/*
24163859Sml29623 			 * Buffer can be reused once the free function
24173859Sml29623 			 * is called.
24183859Sml29623 			 */
24193859Sml29623 			rx_msg_p->max_usage_cnt = 1;
24203859Sml29623 			buffer_free = B_TRUE;
24213859Sml29623 		} else {
24223859Sml29623 			rx_msg_p->max_usage_cnt = rx_msg_p->block_size/bsize;
24233859Sml29623 			if (rx_msg_p->max_usage_cnt == 1) {
24243859Sml29623 				buffer_free = B_TRUE;
24253859Sml29623 			}
24263859Sml29623 		}
24273859Sml29623 	} else {
24283859Sml29623 		rx_msg_p->cur_usage_cnt++;
24293859Sml29623 		if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) {
24303859Sml29623 			buffer_free = B_TRUE;
24313859Sml29623 		}
24323859Sml29623 	}
24333859Sml29623 
24343859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
24353859Sml29623 	    "msgbuf index = %d l2len %d bytes usage %d max_usage %d ",
24366929Smisaki 	    msg_index, l2_len,
24376929Smisaki 	    rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt));
24383859Sml29623 
24396028Ssbehera 	if ((error_type) || (dcf_err) || (pkt_too_long_err)) {
24403859Sml29623 		rdc_stats->ierrors++;
24413859Sml29623 		if (dcf_err) {
24423859Sml29623 			rdc_stats->dcf_err++;
24433859Sml29623 #ifdef	NXGE_DEBUG
24443859Sml29623 			if (!rdc_stats->dcf_err) {
24453859Sml29623 				NXGE_DEBUG_MSG((nxgep, RX_CTL,
24463859Sml29623 				"nxge_receive_packet: channel %d dcf_err rcr"
24473859Sml29623 				" 0x%llx", channel, rcr_entry));
24483859Sml29623 			}
24493859Sml29623 #endif
24503859Sml29623 			NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL,
24516929Smisaki 			    NXGE_FM_EREPORT_RDMC_DCF_ERR);
24526028Ssbehera 		} else if (pkt_too_long_err) {
24536028Ssbehera 			rdc_stats->pkt_too_long_err++;
24546028Ssbehera 			NXGE_DEBUG_MSG((nxgep, RX_CTL, " nxge_receive_packet:"
24556028Ssbehera 			    " channel %d packet length [%d] > "
24566028Ssbehera 			    "maxframesize [%d]", channel, l2_len + ETHERFCSL,
24576028Ssbehera 			    nxgep->mac.maxframesize));
24583859Sml29623 		} else {
24593859Sml29623 				/* Update error stats */
24603859Sml29623 			error_disp_cnt = NXGE_ERROR_SHOW_MAX;
24613859Sml29623 			rdc_stats->errlog.compl_err_type = error_type;
24623859Sml29623 
24633859Sml29623 			switch (error_type) {
24645523Syc148097 			/*
24655523Syc148097 			 * Do not send FMA ereport for RCR_L2_ERROR and
24665523Syc148097 			 * RCR_L4_CSUM_ERROR because most likely they indicate
24675523Syc148097 			 * back pressure rather than HW failures.
24685523Syc148097 			 */
24695165Syc148097 			case RCR_L2_ERROR:
24705165Syc148097 				rdc_stats->l2_err++;
24715165Syc148097 				if (rdc_stats->l2_err <
24725165Syc148097 				    error_disp_cnt) {
24735165Syc148097 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
24745165Syc148097 					    " nxge_receive_packet:"
24755165Syc148097 					    " channel %d RCR L2_ERROR",
24765165Syc148097 					    channel));
24775165Syc148097 				}
24785165Syc148097 				break;
24795165Syc148097 			case RCR_L4_CSUM_ERROR:
24805165Syc148097 				error_send_up = B_TRUE;
24815165Syc148097 				rdc_stats->l4_cksum_err++;
24825165Syc148097 				if (rdc_stats->l4_cksum_err <
24835165Syc148097 				    error_disp_cnt) {
24843859Sml29623 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
24855165Syc148097 					    " nxge_receive_packet:"
24865165Syc148097 					    " channel %d"
24875165Syc148097 					    " RCR L4_CSUM_ERROR", channel));
24885165Syc148097 				}
24895165Syc148097 				break;
24905523Syc148097 			/*
24915523Syc148097 			 * Do not send FMA ereport for RCR_FFLP_SOFT_ERROR and
24925523Syc148097 			 * RCR_ZCP_SOFT_ERROR because they reflect the same
24935523Syc148097 			 * FFLP and ZCP errors that have been reported by
24945523Syc148097 			 * nxge_fflp.c and nxge_zcp.c.
24955523Syc148097 			 */
24965165Syc148097 			case RCR_FFLP_SOFT_ERROR:
24975165Syc148097 				error_send_up = B_TRUE;
24985165Syc148097 				rdc_stats->fflp_soft_err++;
24995165Syc148097 				if (rdc_stats->fflp_soft_err <
25005165Syc148097 				    error_disp_cnt) {
25015165Syc148097 					NXGE_ERROR_MSG((nxgep,
25025165Syc148097 					    NXGE_ERR_CTL,
25035165Syc148097 					    " nxge_receive_packet:"
25045165Syc148097 					    " channel %d"
25055165Syc148097 					    " RCR FFLP_SOFT_ERROR", channel));
25065165Syc148097 				}
25075165Syc148097 				break;
25085165Syc148097 			case RCR_ZCP_SOFT_ERROR:
25095165Syc148097 				error_send_up = B_TRUE;
25105165Syc148097 				rdc_stats->fflp_soft_err++;
25115165Syc148097 				if (rdc_stats->zcp_soft_err <
25125165Syc148097 				    error_disp_cnt)
25135165Syc148097 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
25145165Syc148097 					    " nxge_receive_packet: Channel %d"
25155165Syc148097 					    " RCR ZCP_SOFT_ERROR", channel));
25165165Syc148097 				break;
25175165Syc148097 			default:
25185165Syc148097 				rdc_stats->rcr_unknown_err++;
25195165Syc148097 				if (rdc_stats->rcr_unknown_err
25205165Syc148097 				    < error_disp_cnt) {
25215165Syc148097 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
25225165Syc148097 					    " nxge_receive_packet: Channel %d"
25235165Syc148097 					    " RCR entry 0x%llx error 0x%x",
25245165Syc148097 					    rcr_entry, channel, error_type));
25255165Syc148097 				}
25265165Syc148097 				break;
25273859Sml29623 			}
25283859Sml29623 		}
25293859Sml29623 
25303859Sml29623 		/*
25313859Sml29623 		 * Update and repost buffer block if max usage
25323859Sml29623 		 * count is reached.
25333859Sml29623 		 */
25343859Sml29623 		if (error_send_up == B_FALSE) {
25354874Sml29623 			atomic_inc_32(&rx_msg_p->ref_cnt);
25363859Sml29623 			if (buffer_free == B_TRUE) {
25373859Sml29623 				rx_msg_p->free = B_TRUE;
25383859Sml29623 			}
25393859Sml29623 
25403859Sml29623 			MUTEX_EXIT(&rx_rbr_p->lock);
25413859Sml29623 			MUTEX_EXIT(&rcr_p->lock);
25423859Sml29623 			nxge_freeb(rx_msg_p);
25433859Sml29623 			return;
25443859Sml29623 		}
25453859Sml29623 	}
25463859Sml29623 
25473859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
25486929Smisaki 	    "==> nxge_receive_packet: DMA sync second "));
25493859Sml29623 
25505165Syc148097 	bytes_read = rcr_p->rcvd_pkt_bytes;
25513859Sml29623 	skip_len = sw_offset_bytes + hdr_size;
25523859Sml29623 	if (!rx_msg_p->rx_use_bcopy) {
25534874Sml29623 		/*
25544874Sml29623 		 * For loaned up buffers, the driver reference count
25554874Sml29623 		 * will be incremented first and then the free state.
25564874Sml29623 		 */
25575165Syc148097 		if ((nmp = nxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) {
25585165Syc148097 			if (first_entry) {
25595165Syc148097 				nmp->b_rptr = &nmp->b_rptr[skip_len];
25605165Syc148097 				if (l2_len < bsize - skip_len) {
25615165Syc148097 					nmp->b_wptr = &nmp->b_rptr[l2_len];
25625165Syc148097 				} else {
25635165Syc148097 					nmp->b_wptr = &nmp->b_rptr[bsize
25645165Syc148097 					    - skip_len];
25655165Syc148097 				}
25665165Syc148097 			} else {
25675165Syc148097 				if (l2_len - bytes_read < bsize) {
25685165Syc148097 					nmp->b_wptr =
25695165Syc148097 					    &nmp->b_rptr[l2_len - bytes_read];
25705165Syc148097 				} else {
25715165Syc148097 					nmp->b_wptr = &nmp->b_rptr[bsize];
25725165Syc148097 				}
25735165Syc148097 			}
25745165Syc148097 		}
25753859Sml29623 	} else {
25765165Syc148097 		if (first_entry) {
25775165Syc148097 			nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len,
25785165Syc148097 			    l2_len < bsize - skip_len ?
25795165Syc148097 			    l2_len : bsize - skip_len);
25805165Syc148097 		} else {
25815165Syc148097 			nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset,
25825165Syc148097 			    l2_len - bytes_read < bsize ?
25835165Syc148097 			    l2_len - bytes_read : bsize);
25845165Syc148097 		}
25853859Sml29623 	}
25863859Sml29623 	if (nmp != NULL) {
25877145Syc148097 		if (first_entry) {
25887145Syc148097 			/*
25897145Syc148097 			 * Jumbo packets may be received with more than one
25907145Syc148097 			 * buffer, increment ipackets for the first entry only.
25917145Syc148097 			 */
25927145Syc148097 			rdc_stats->ipackets++;
25937145Syc148097 
25947145Syc148097 			/* Update ibytes for kstat. */
25957145Syc148097 			rdc_stats->ibytes += skip_len
25967145Syc148097 			    + l2_len < bsize ? l2_len : bsize;
25977145Syc148097 			/*
25987145Syc148097 			 * Update the number of bytes read so far for the
25997145Syc148097 			 * current frame.
26007145Syc148097 			 */
26015165Syc148097 			bytes_read  = nmp->b_wptr - nmp->b_rptr;
26027145Syc148097 		} else {
26037145Syc148097 			rdc_stats->ibytes += l2_len - bytes_read < bsize ?
26047145Syc148097 			    l2_len - bytes_read : bsize;
26053859Sml29623 			bytes_read += nmp->b_wptr - nmp->b_rptr;
26067145Syc148097 		}
26075165Syc148097 
26085165Syc148097 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
26095165Syc148097 		    "==> nxge_receive_packet after dupb: "
26105165Syc148097 		    "rbr consumed %d "
26115165Syc148097 		    "pktbufsz_type %d "
26125165Syc148097 		    "nmp $%p rptr $%p wptr $%p "
26135165Syc148097 		    "buf_offset %d bzise %d l2_len %d skip_len %d",
26145165Syc148097 		    rx_rbr_p->rbr_consumed,
26155165Syc148097 		    pktbufsz_type,
26165165Syc148097 		    nmp, nmp->b_rptr, nmp->b_wptr,
26175165Syc148097 		    buf_offset, bsize, l2_len, skip_len));
26183859Sml29623 	} else {
26193859Sml29623 		cmn_err(CE_WARN, "!nxge_receive_packet: "
26206929Smisaki 		    "update stats (error)");
26214977Sraghus 		atomic_inc_32(&rx_msg_p->ref_cnt);
26224977Sraghus 		if (buffer_free == B_TRUE) {
26234977Sraghus 			rx_msg_p->free = B_TRUE;
26244977Sraghus 		}
26254977Sraghus 		MUTEX_EXIT(&rx_rbr_p->lock);
26264977Sraghus 		MUTEX_EXIT(&rcr_p->lock);
26274977Sraghus 		nxge_freeb(rx_msg_p);
26284977Sraghus 		return;
26293859Sml29623 	}
26305060Syc148097 
26313859Sml29623 	if (buffer_free == B_TRUE) {
26323859Sml29623 		rx_msg_p->free = B_TRUE;
26333859Sml29623 	}
26347145Syc148097 
26353859Sml29623 	is_valid = (nmp != NULL);
26365165Syc148097 
26375165Syc148097 	rcr_p->rcvd_pkt_bytes = bytes_read;
26385165Syc148097 
26393859Sml29623 	MUTEX_EXIT(&rx_rbr_p->lock);
26403859Sml29623 	MUTEX_EXIT(&rcr_p->lock);
26413859Sml29623 
26423859Sml29623 	if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) {
26433859Sml29623 		atomic_inc_32(&rx_msg_p->ref_cnt);
26443859Sml29623 		nxge_freeb(rx_msg_p);
26453859Sml29623 	}
26463859Sml29623 
26473859Sml29623 	if (is_valid) {
26483859Sml29623 		nmp->b_cont = NULL;
26493859Sml29623 		if (first_entry) {
26503859Sml29623 			*mp = nmp;
26513859Sml29623 			*mp_cont = NULL;
26525165Syc148097 		} else {
26533859Sml29623 			*mp_cont = nmp;
26545165Syc148097 		}
26553859Sml29623 	}
26563859Sml29623 
26573859Sml29623 	/*
26587145Syc148097 	 * ERROR, FRAG and PKT_TYPE are only reported in the first entry.
26597145Syc148097 	 * If a packet is not fragmented and no error bit is set, then
26607145Syc148097 	 * L4 checksum is OK.
26613859Sml29623 	 */
26627145Syc148097 
26633859Sml29623 	if (is_valid && !multi) {
26646495Sspeer 		/*
26657145Syc148097 		 * Update hardware checksuming.
26667145Syc148097 		 *
26676611Sml29623 		 * If the checksum flag nxge_chksum_offload
26686611Sml29623 		 * is 1, TCP and UDP packets can be sent
26696495Sspeer 		 * up with good checksum. If the checksum flag
26706611Sml29623 		 * is set to 0, checksum reporting will apply to
26716495Sspeer 		 * TCP packets only (workaround for a hardware bug).
26726611Sml29623 		 * If the checksum flag nxge_cksum_offload is
26736611Sml29623 		 * greater than 1, both TCP and UDP packets
26746611Sml29623 		 * will not be reported its hardware checksum results.
26756495Sspeer 		 */
26766611Sml29623 		if (nxge_cksum_offload == 1) {
26776495Sspeer 			is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP ||
26786929Smisaki 			    pkt_type == RCR_PKT_IS_UDP) ?
26796929Smisaki 			    B_TRUE: B_FALSE);
26806611Sml29623 		} else if (!nxge_cksum_offload) {
26816495Sspeer 			/* TCP checksum only. */
26826495Sspeer 			is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP) ?
26836929Smisaki 			    B_TRUE: B_FALSE);
26846495Sspeer 		}
26853859Sml29623 
26863859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_receive_packet: "
26876929Smisaki 		    "is_valid 0x%x multi 0x%llx pkt %d frag %d error %d",
26886929Smisaki 		    is_valid, multi, is_tcp_udp, frag, error_type));
26893859Sml29623 
26903859Sml29623 		if (is_tcp_udp && !frag && !error_type) {
26913859Sml29623 			(void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0,
26926929Smisaki 			    HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0);
26933859Sml29623 			NXGE_DEBUG_MSG((nxgep, RX_CTL,
26946929Smisaki 			    "==> nxge_receive_packet: Full tcp/udp cksum "
26956929Smisaki 			    "is_valid 0x%x multi 0x%llx pkt %d frag %d "
26966929Smisaki 			    "error %d",
26976929Smisaki 			    is_valid, multi, is_tcp_udp, frag, error_type));
26983859Sml29623 		}
26993859Sml29623 	}
27003859Sml29623 
27013859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
27026929Smisaki 	    "==> nxge_receive_packet: *mp 0x%016llx", *mp));
27033859Sml29623 
27043859Sml29623 	*multi_p = (multi == RCR_MULTI_MASK);
27053859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_receive_packet: "
27066929Smisaki 	    "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx",
27076929Smisaki 	    *multi_p, nmp, *mp, *mp_cont));
27083859Sml29623 }
27093859Sml29623 
27103859Sml29623 /*ARGSUSED*/
27113859Sml29623 static nxge_status_t
27126495Sspeer nxge_rx_err_evnts(p_nxge_t nxgep, int channel, rx_dma_ctl_stat_t cs)
27133859Sml29623 {
27143859Sml29623 	p_nxge_rx_ring_stats_t	rdc_stats;
27153859Sml29623 	npi_handle_t		handle;
27163859Sml29623 	npi_status_t		rs;
27173859Sml29623 	boolean_t		rxchan_fatal = B_FALSE;
27183859Sml29623 	boolean_t		rxport_fatal = B_FALSE;
27193859Sml29623 	uint8_t			portn;
27203859Sml29623 	nxge_status_t		status = NXGE_OK;
27213859Sml29623 	uint32_t		error_disp_cnt = NXGE_ERROR_SHOW_MAX;
27223859Sml29623 	NXGE_DEBUG_MSG((nxgep, INT_CTL, "==> nxge_rx_err_evnts"));
27233859Sml29623 
27243859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
27253859Sml29623 	portn = nxgep->mac.portnum;
27266495Sspeer 	rdc_stats = &nxgep->statsp->rdc_stats[channel];
27273859Sml29623 
27283859Sml29623 	if (cs.bits.hdw.rbr_tmout) {
27293859Sml29623 		rdc_stats->rx_rbr_tmout++;
27303859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
27316929Smisaki 		    NXGE_FM_EREPORT_RDMC_RBR_TMOUT);
27323859Sml29623 		rxchan_fatal = B_TRUE;
27333859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27346929Smisaki 		    "==> nxge_rx_err_evnts: rx_rbr_timeout"));
27353859Sml29623 	}
27363859Sml29623 	if (cs.bits.hdw.rsp_cnt_err) {
27373859Sml29623 		rdc_stats->rsp_cnt_err++;
27383859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
27396929Smisaki 		    NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR);
27403859Sml29623 		rxchan_fatal = B_TRUE;
27413859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27426929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
27436929Smisaki 		    "rsp_cnt_err", channel));
27443859Sml29623 	}
27453859Sml29623 	if (cs.bits.hdw.byte_en_bus) {
27463859Sml29623 		rdc_stats->byte_en_bus++;
27473859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
27486929Smisaki 		    NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS);
27493859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27506929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
27516929Smisaki 		    "fatal error: byte_en_bus", channel));
27523859Sml29623 		rxchan_fatal = B_TRUE;
27533859Sml29623 	}
27543859Sml29623 	if (cs.bits.hdw.rsp_dat_err) {
27553859Sml29623 		rdc_stats->rsp_dat_err++;
27563859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
27576929Smisaki 		    NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR);
27583859Sml29623 		rxchan_fatal = B_TRUE;
27593859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27606929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
27616929Smisaki 		    "fatal error: rsp_dat_err", channel));
27623859Sml29623 	}
27633859Sml29623 	if (cs.bits.hdw.rcr_ack_err) {
27643859Sml29623 		rdc_stats->rcr_ack_err++;
27653859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
27666929Smisaki 		    NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR);
27673859Sml29623 		rxchan_fatal = B_TRUE;
27683859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27696929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
27706929Smisaki 		    "fatal error: rcr_ack_err", channel));
27713859Sml29623 	}
27723859Sml29623 	if (cs.bits.hdw.dc_fifo_err) {
27733859Sml29623 		rdc_stats->dc_fifo_err++;
27743859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
27756929Smisaki 		    NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR);
27763859Sml29623 		/* This is not a fatal error! */
27773859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27786929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
27796929Smisaki 		    "dc_fifo_err", channel));
27803859Sml29623 		rxport_fatal = B_TRUE;
27813859Sml29623 	}
27823859Sml29623 	if ((cs.bits.hdw.rcr_sha_par) || (cs.bits.hdw.rbr_pre_par)) {
27833859Sml29623 		if ((rs = npi_rxdma_ring_perr_stat_get(handle,
27846929Smisaki 		    &rdc_stats->errlog.pre_par,
27856929Smisaki 		    &rdc_stats->errlog.sha_par))
27866929Smisaki 		    != NPI_SUCCESS) {
27873859Sml29623 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27886929Smisaki 			    "==> nxge_rx_err_evnts(channel %d): "
27896929Smisaki 			    "rcr_sha_par: get perr", channel));
27903859Sml29623 			return (NXGE_ERROR | rs);
27913859Sml29623 		}
27923859Sml29623 		if (cs.bits.hdw.rcr_sha_par) {
27933859Sml29623 			rdc_stats->rcr_sha_par++;
27943859Sml29623 			NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
27956929Smisaki 			    NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR);
27963859Sml29623 			rxchan_fatal = B_TRUE;
27973859Sml29623 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
27986929Smisaki 			    "==> nxge_rx_err_evnts(channel %d): "
27996929Smisaki 			    "fatal error: rcr_sha_par", channel));
28003859Sml29623 		}
28013859Sml29623 		if (cs.bits.hdw.rbr_pre_par) {
28023859Sml29623 			rdc_stats->rbr_pre_par++;
28033859Sml29623 			NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
28046929Smisaki 			    NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR);
28053859Sml29623 			rxchan_fatal = B_TRUE;
28063859Sml29623 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28076929Smisaki 			    "==> nxge_rx_err_evnts(channel %d): "
28086929Smisaki 			    "fatal error: rbr_pre_par", channel));
28093859Sml29623 		}
28103859Sml29623 	}
28116172Syc148097 	/*
28126172Syc148097 	 * The Following 4 status bits are for information, the system
28136172Syc148097 	 * is running fine. There is no need to send FMA ereports or
28146172Syc148097 	 * log messages.
28156172Syc148097 	 */
28163859Sml29623 	if (cs.bits.hdw.port_drop_pkt) {
28173859Sml29623 		rdc_stats->port_drop_pkt++;
28183859Sml29623 	}
28193859Sml29623 	if (cs.bits.hdw.wred_drop) {
28203859Sml29623 		rdc_stats->wred_drop++;
28213859Sml29623 	}
28223859Sml29623 	if (cs.bits.hdw.rbr_pre_empty) {
28233859Sml29623 		rdc_stats->rbr_pre_empty++;
28243859Sml29623 	}
28253859Sml29623 	if (cs.bits.hdw.rcr_shadow_full) {
28263859Sml29623 		rdc_stats->rcr_shadow_full++;
28273859Sml29623 	}
28283859Sml29623 	if (cs.bits.hdw.config_err) {
28293859Sml29623 		rdc_stats->config_err++;
28303859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
28316929Smisaki 		    NXGE_FM_EREPORT_RDMC_CONFIG_ERR);
28323859Sml29623 		rxchan_fatal = B_TRUE;
28333859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28346929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
28356929Smisaki 		    "config error", channel));
28363859Sml29623 	}
28373859Sml29623 	if (cs.bits.hdw.rcrincon) {
28383859Sml29623 		rdc_stats->rcrincon++;
28393859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
28406929Smisaki 		    NXGE_FM_EREPORT_RDMC_RCRINCON);
28413859Sml29623 		rxchan_fatal = B_TRUE;
28423859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28436929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
28446929Smisaki 		    "fatal error: rcrincon error", channel));
28453859Sml29623 	}
28463859Sml29623 	if (cs.bits.hdw.rcrfull) {
28473859Sml29623 		rdc_stats->rcrfull++;
28483859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
28496929Smisaki 		    NXGE_FM_EREPORT_RDMC_RCRFULL);
28503859Sml29623 		rxchan_fatal = B_TRUE;
28513859Sml29623 		if (rdc_stats->rcrfull < error_disp_cnt)
28523859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28536929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
28546929Smisaki 		    "fatal error: rcrfull error", channel));
28553859Sml29623 	}
28563859Sml29623 	if (cs.bits.hdw.rbr_empty) {
28576172Syc148097 		/*
28586172Syc148097 		 * This bit is for information, there is no need
28596172Syc148097 		 * send FMA ereport or log a message.
28606172Syc148097 		 */
28613859Sml29623 		rdc_stats->rbr_empty++;
28623859Sml29623 	}
28633859Sml29623 	if (cs.bits.hdw.rbrfull) {
28643859Sml29623 		rdc_stats->rbrfull++;
28653859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
28666929Smisaki 		    NXGE_FM_EREPORT_RDMC_RBRFULL);
28673859Sml29623 		rxchan_fatal = B_TRUE;
28683859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28696929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
28706929Smisaki 		    "fatal error: rbr_full error", channel));
28713859Sml29623 	}
28723859Sml29623 	if (cs.bits.hdw.rbrlogpage) {
28733859Sml29623 		rdc_stats->rbrlogpage++;
28743859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
28756929Smisaki 		    NXGE_FM_EREPORT_RDMC_RBRLOGPAGE);
28763859Sml29623 		rxchan_fatal = B_TRUE;
28773859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28786929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
28796929Smisaki 		    "fatal error: rbr logical page error", channel));
28803859Sml29623 	}
28813859Sml29623 	if (cs.bits.hdw.cfiglogpage) {
28823859Sml29623 		rdc_stats->cfiglogpage++;
28833859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
28846929Smisaki 		    NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE);
28853859Sml29623 		rxchan_fatal = B_TRUE;
28863859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28876929Smisaki 		    "==> nxge_rx_err_evnts(channel %d): "
28886929Smisaki 		    "fatal error: cfig logical page error", channel));
28893859Sml29623 	}
28903859Sml29623 
28913859Sml29623 	if (rxport_fatal)  {
28923859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
28936495Sspeer 		    " nxge_rx_err_evnts: fatal error on Port #%d\n",
28946495Sspeer 		    portn));
28956495Sspeer 		if (isLDOMguest(nxgep)) {
28966495Sspeer 			status = NXGE_ERROR;
28976495Sspeer 		} else {
28986495Sspeer 			status = nxge_ipp_fatal_err_recover(nxgep);
28996495Sspeer 			if (status == NXGE_OK) {
29006495Sspeer 				FM_SERVICE_RESTORED(nxgep);
29016495Sspeer 			}
29023859Sml29623 		}
29033859Sml29623 	}
29043859Sml29623 
29053859Sml29623 	if (rxchan_fatal) {
29063859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
29076495Sspeer 		    " nxge_rx_err_evnts: fatal error on Channel #%d\n",
29086495Sspeer 		    channel));
29096495Sspeer 		if (isLDOMguest(nxgep)) {
29106495Sspeer 			status = NXGE_ERROR;
29116495Sspeer 		} else {
29126495Sspeer 			status = nxge_rxdma_fatal_err_recover(nxgep, channel);
29136495Sspeer 			if (status == NXGE_OK) {
29146495Sspeer 				FM_SERVICE_RESTORED(nxgep);
29156495Sspeer 			}
29163859Sml29623 		}
29173859Sml29623 	}
29183859Sml29623 
29193859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rx_err_evnts"));
29203859Sml29623 
29213859Sml29623 	return (status);
29223859Sml29623 }
29233859Sml29623 
29246495Sspeer /*
29256495Sspeer  * nxge_rdc_hvio_setup
29266495Sspeer  *
29276495Sspeer  *	This code appears to setup some Hypervisor variables.
29286495Sspeer  *
29296495Sspeer  * Arguments:
29306495Sspeer  * 	nxgep
29316495Sspeer  * 	channel
29326495Sspeer  *
29336495Sspeer  * Notes:
29346495Sspeer  *	What does NIU_LP_WORKAROUND mean?
29356495Sspeer  *
29366495Sspeer  * NPI/NXGE function calls:
29376495Sspeer  *	na
29386495Sspeer  *
29396495Sspeer  * Context:
29406495Sspeer  *	Any domain
29416495Sspeer  */
29426495Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
29436495Sspeer static void
29446495Sspeer nxge_rdc_hvio_setup(
29456495Sspeer 	nxge_t *nxgep, int channel)
29463859Sml29623 {
29476495Sspeer 	nxge_dma_common_t	*dma_common;
29486495Sspeer 	nxge_dma_common_t	*dma_control;
29496495Sspeer 	rx_rbr_ring_t		*ring;
29506495Sspeer 
29516495Sspeer 	ring = nxgep->rx_rbr_rings->rbr_rings[channel];
29526495Sspeer 	dma_common = nxgep->rx_buf_pool_p->dma_buf_pool_p[channel];
29536495Sspeer 
29546495Sspeer 	ring->hv_set = B_FALSE;
29556495Sspeer 
29566495Sspeer 	ring->hv_rx_buf_base_ioaddr_pp = (uint64_t)
29576495Sspeer 	    dma_common->orig_ioaddr_pp;
29586495Sspeer 	ring->hv_rx_buf_ioaddr_size = (uint64_t)
29596495Sspeer 	    dma_common->orig_alength;
29606495Sspeer 
29616495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: "
29626495Sspeer 	    "channel %d data buf base io $%lx ($%p) size 0x%lx (%ld 0x%lx)",
29636495Sspeer 	    channel, ring->hv_rx_buf_base_ioaddr_pp,
29646495Sspeer 	    dma_common->ioaddr_pp, ring->hv_rx_buf_ioaddr_size,
29656495Sspeer 	    dma_common->orig_alength, dma_common->orig_alength));
29666495Sspeer 
29676495Sspeer 	dma_control = nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel];
29686495Sspeer 
29696495Sspeer 	ring->hv_rx_cntl_base_ioaddr_pp =
29706495Sspeer 	    (uint64_t)dma_control->orig_ioaddr_pp;
29716495Sspeer 	ring->hv_rx_cntl_ioaddr_size =
29726495Sspeer 	    (uint64_t)dma_control->orig_alength;
29736495Sspeer 
29746495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: "
29756495Sspeer 	    "channel %d cntl base io $%p ($%p) size 0x%llx (%d 0x%x)",
29766495Sspeer 	    channel, ring->hv_rx_cntl_base_ioaddr_pp,
29776495Sspeer 	    dma_control->ioaddr_pp, ring->hv_rx_cntl_ioaddr_size,
29786495Sspeer 	    dma_control->orig_alength, dma_control->orig_alength));
29796495Sspeer }
29803859Sml29623 #endif
29813859Sml29623 
29826495Sspeer /*
29836495Sspeer  * nxge_map_rxdma
29846495Sspeer  *
29856495Sspeer  *	Map an RDC into our kernel space.
29866495Sspeer  *
29876495Sspeer  * Arguments:
29886495Sspeer  * 	nxgep
29896495Sspeer  * 	channel	The channel to map.
29906495Sspeer  *
29916495Sspeer  * Notes:
29926495Sspeer  *	1. Allocate & initialise a memory pool, if necessary.
29936495Sspeer  *	2. Allocate however many receive buffers are required.
29946495Sspeer  *	3. Setup buffers, descriptors, and mailbox.
29956495Sspeer  *
29966495Sspeer  * NPI/NXGE function calls:
29976495Sspeer  *	nxge_alloc_rx_mem_pool()
29986495Sspeer  *	nxge_alloc_rbb()
29996495Sspeer  *	nxge_map_rxdma_channel()
30006495Sspeer  *
30016495Sspeer  * Registers accessed:
30026495Sspeer  *
30036495Sspeer  * Context:
30046495Sspeer  *	Any domain
30056495Sspeer  */
30066495Sspeer static nxge_status_t
30076495Sspeer nxge_map_rxdma(p_nxge_t nxgep, int channel)
30086495Sspeer {
30096495Sspeer 	nxge_dma_common_t	**data;
30106495Sspeer 	nxge_dma_common_t	**control;
30116495Sspeer 	rx_rbr_ring_t		**rbr_ring;
30126495Sspeer 	rx_rcr_ring_t		**rcr_ring;
30136495Sspeer 	rx_mbox_t		**mailbox;
30146495Sspeer 	uint32_t		chunks;
30156495Sspeer 
30166495Sspeer 	nxge_status_t		status;
30176495Sspeer 
30183859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma"));
30193859Sml29623 
30206495Sspeer 	if (!nxgep->rx_buf_pool_p) {
30216495Sspeer 		if (nxge_alloc_rx_mem_pool(nxgep) != NXGE_OK) {
30226495Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
30236495Sspeer 			    "<== nxge_map_rxdma: buf not allocated"));
30246495Sspeer 			return (NXGE_ERROR);
30256495Sspeer 		}
30263859Sml29623 	}
30273859Sml29623 
30286495Sspeer 	if (nxge_alloc_rxb(nxgep, channel) != NXGE_OK)
30296495Sspeer 		return (NXGE_ERROR);
30303859Sml29623 
30313859Sml29623 	/*
30323859Sml29623 	 * Timeout should be set based on the system clock divider.
30333859Sml29623 	 * The following timeout value of 1 assumes that the
30343859Sml29623 	 * granularity (1000) is 3 microseconds running at 300MHz.
30353859Sml29623 	 */
30363859Sml29623 
30373859Sml29623 	nxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT;
30383859Sml29623 	nxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT;
30393859Sml29623 
30403859Sml29623 	/*
30416495Sspeer 	 * Map descriptors from the buffer polls for each dma channel.
30426495Sspeer 	 */
30436495Sspeer 
30446495Sspeer 	/*
30456495Sspeer 	 * Set up and prepare buffer blocks, descriptors
30466495Sspeer 	 * and mailbox.
30473859Sml29623 	 */
30486495Sspeer 	data = &nxgep->rx_buf_pool_p->dma_buf_pool_p[channel];
30496495Sspeer 	rbr_ring = &nxgep->rx_rbr_rings->rbr_rings[channel];
30506495Sspeer 	chunks = nxgep->rx_buf_pool_p->num_chunks[channel];
30516495Sspeer 
30526495Sspeer 	control = &nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel];
30536495Sspeer 	rcr_ring = &nxgep->rx_rcr_rings->rcr_rings[channel];
30546495Sspeer 
30556495Sspeer 	mailbox = &nxgep->rx_mbox_areas_p->rxmbox_areas[channel];
30566495Sspeer 
30576495Sspeer 	status = nxge_map_rxdma_channel(nxgep, channel, data, rbr_ring,
30586495Sspeer 	    chunks, control, rcr_ring, mailbox);
30596495Sspeer 	if (status != NXGE_OK) {
30606495Sspeer 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
30616929Smisaki 		    "==> nxge_map_rxdma: nxge_map_rxdma_channel(%d) "
30626929Smisaki 		    "returned 0x%x",
30636929Smisaki 		    channel, status));
30646495Sspeer 		return (status);
30656495Sspeer 	}
30666495Sspeer 	nxgep->rx_rbr_rings->rbr_rings[channel]->index = (uint16_t)channel;
30676495Sspeer 	nxgep->rx_rcr_rings->rcr_rings[channel]->index = (uint16_t)channel;
30686495Sspeer 	nxgep->rx_rcr_rings->rcr_rings[channel]->rdc_stats =
30696495Sspeer 	    &nxgep->statsp->rdc_stats[channel];
30703859Sml29623 
30713859Sml29623 #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
30726495Sspeer 	if (!isLDOMguest(nxgep))
30736495Sspeer 		nxge_rdc_hvio_setup(nxgep, channel);
30746495Sspeer #endif
30756495Sspeer 
30763859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
30776495Sspeer 	    "<== nxge_map_rxdma: (status 0x%x channel %d)", status, channel));
30783859Sml29623 
30793859Sml29623 	return (status);
30803859Sml29623 }
30813859Sml29623 
30823859Sml29623 static void
30836495Sspeer nxge_unmap_rxdma(p_nxge_t nxgep, int channel)
30843859Sml29623 {
30856495Sspeer 	rx_rbr_ring_t	*rbr_ring;
30866495Sspeer 	rx_rcr_ring_t	*rcr_ring;
30876495Sspeer 	rx_mbox_t	*mailbox;
30886495Sspeer 
30896495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_unmap_rxdma(%d)", channel));
30906495Sspeer 
30916495Sspeer 	if (!nxgep->rx_rbr_rings || !nxgep->rx_rcr_rings ||
30926495Sspeer 	    !nxgep->rx_mbox_areas_p)
30933859Sml29623 		return;
30946495Sspeer 
30956495Sspeer 	rbr_ring = nxgep->rx_rbr_rings->rbr_rings[channel];
30966495Sspeer 	rcr_ring = nxgep->rx_rcr_rings->rcr_rings[channel];
30976495Sspeer 	mailbox = nxgep->rx_mbox_areas_p->rxmbox_areas[channel];
30986495Sspeer 
30996495Sspeer 	if (!rbr_ring || !rcr_ring || !mailbox)
31003859Sml29623 		return;
31016495Sspeer 
31026495Sspeer 	(void) nxge_unmap_rxdma_channel(
31036929Smisaki 	    nxgep, channel, rbr_ring, rcr_ring, mailbox);
31046495Sspeer 
31056495Sspeer 	nxge_free_rxb(nxgep, channel);
31066495Sspeer 
31076495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma"));
31083859Sml29623 }
31093859Sml29623 
31103859Sml29623 nxge_status_t
31113859Sml29623 nxge_map_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
31123859Sml29623     p_nxge_dma_common_t *dma_buf_p,  p_rx_rbr_ring_t *rbr_p,
31133859Sml29623     uint32_t num_chunks,
31143859Sml29623     p_nxge_dma_common_t *dma_cntl_p, p_rx_rcr_ring_t *rcr_p,
31153859Sml29623     p_rx_mbox_t *rx_mbox_p)
31163859Sml29623 {
31173859Sml29623 	int	status = NXGE_OK;
31183859Sml29623 
31193859Sml29623 	/*
31203859Sml29623 	 * Set up and prepare buffer blocks, descriptors
31213859Sml29623 	 * and mailbox.
31223859Sml29623 	 */
31233859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
31246929Smisaki 	    "==> nxge_map_rxdma_channel (channel %d)", channel));
31253859Sml29623 	/*
31263859Sml29623 	 * Receive buffer blocks
31273859Sml29623 	 */
31283859Sml29623 	status = nxge_map_rxdma_channel_buf_ring(nxgep, channel,
31296929Smisaki 	    dma_buf_p, rbr_p, num_chunks);
31303859Sml29623 	if (status != NXGE_OK) {
31313859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
31326929Smisaki 		    "==> nxge_map_rxdma_channel (channel %d): "
31336929Smisaki 		    "map buffer failed 0x%x", channel, status));
31343859Sml29623 		goto nxge_map_rxdma_channel_exit;
31353859Sml29623 	}
31363859Sml29623 
31373859Sml29623 	/*
31383859Sml29623 	 * Receive block ring, completion ring and mailbox.
31393859Sml29623 	 */
31403859Sml29623 	status = nxge_map_rxdma_channel_cfg_ring(nxgep, channel,
31416929Smisaki 	    dma_cntl_p, rbr_p, rcr_p, rx_mbox_p);
31423859Sml29623 	if (status != NXGE_OK) {
31433859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
31446929Smisaki 		    "==> nxge_map_rxdma_channel (channel %d): "
31456929Smisaki 		    "map config failed 0x%x", channel, status));
31463859Sml29623 		goto nxge_map_rxdma_channel_fail2;
31473859Sml29623 	}
31483859Sml29623 
31493859Sml29623 	goto nxge_map_rxdma_channel_exit;
31503859Sml29623 
31513859Sml29623 nxge_map_rxdma_channel_fail3:
31523859Sml29623 	/* Free rbr, rcr */
31533859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
31546929Smisaki 	    "==> nxge_map_rxdma_channel: free rbr/rcr "
31556929Smisaki 	    "(status 0x%x channel %d)",
31566929Smisaki 	    status, channel));
31573859Sml29623 	nxge_unmap_rxdma_channel_cfg_ring(nxgep,
31586929Smisaki 	    *rcr_p, *rx_mbox_p);
31593859Sml29623 
31603859Sml29623 nxge_map_rxdma_channel_fail2:
31613859Sml29623 	/* Free buffer blocks */
31623859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
31636929Smisaki 	    "==> nxge_map_rxdma_channel: free rx buffers"
31646929Smisaki 	    "(nxgep 0x%x status 0x%x channel %d)",
31656929Smisaki 	    nxgep, status, channel));
31663859Sml29623 	nxge_unmap_rxdma_channel_buf_ring(nxgep, *rbr_p);
31673859Sml29623 
31684185Sspeer 	status = NXGE_ERROR;
31694185Sspeer 
31703859Sml29623 nxge_map_rxdma_channel_exit:
31713859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
31726929Smisaki 	    "<== nxge_map_rxdma_channel: "
31736929Smisaki 	    "(nxgep 0x%x status 0x%x channel %d)",
31746929Smisaki 	    nxgep, status, channel));
31753859Sml29623 
31763859Sml29623 	return (status);
31773859Sml29623 }
31783859Sml29623 
31793859Sml29623 /*ARGSUSED*/
31803859Sml29623 static void
31813859Sml29623 nxge_unmap_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
31823859Sml29623     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
31833859Sml29623 {
31843859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
31856929Smisaki 	    "==> nxge_unmap_rxdma_channel (channel %d)", channel));
31863859Sml29623 
31873859Sml29623 	/*
31883859Sml29623 	 * unmap receive block ring, completion ring and mailbox.
31893859Sml29623 	 */
31903859Sml29623 	(void) nxge_unmap_rxdma_channel_cfg_ring(nxgep,
31916929Smisaki 	    rcr_p, rx_mbox_p);
31923859Sml29623 
31933859Sml29623 	/* unmap buffer blocks */
31943859Sml29623 	(void) nxge_unmap_rxdma_channel_buf_ring(nxgep, rbr_p);
31953859Sml29623 
31963859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma_channel"));
31973859Sml29623 }
31983859Sml29623 
31993859Sml29623 /*ARGSUSED*/
32003859Sml29623 static nxge_status_t
32013859Sml29623 nxge_map_rxdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel,
32023859Sml29623     p_nxge_dma_common_t *dma_cntl_p, p_rx_rbr_ring_t *rbr_p,
32033859Sml29623     p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p)
32043859Sml29623 {
32053859Sml29623 	p_rx_rbr_ring_t 	rbrp;
32063859Sml29623 	p_rx_rcr_ring_t 	rcrp;
32073859Sml29623 	p_rx_mbox_t 		mboxp;
32083859Sml29623 	p_nxge_dma_common_t 	cntl_dmap;
32093859Sml29623 	p_nxge_dma_common_t 	dmap;
32103859Sml29623 	p_rx_msg_t 		*rx_msg_ring;
32113859Sml29623 	p_rx_msg_t 		rx_msg_p;
32123859Sml29623 	p_rbr_cfig_a_t		rcfga_p;
32133859Sml29623 	p_rbr_cfig_b_t		rcfgb_p;
32143859Sml29623 	p_rcrcfig_a_t		cfga_p;
32153859Sml29623 	p_rcrcfig_b_t		cfgb_p;
32163859Sml29623 	p_rxdma_cfig1_t		cfig1_p;
32173859Sml29623 	p_rxdma_cfig2_t		cfig2_p;
32183859Sml29623 	p_rbr_kick_t		kick_p;
32193859Sml29623 	uint32_t		dmaaddrp;
32203859Sml29623 	uint32_t		*rbr_vaddrp;
32213859Sml29623 	uint32_t		bkaddr;
32223859Sml29623 	nxge_status_t		status = NXGE_OK;
32233859Sml29623 	int			i;
32243859Sml29623 	uint32_t 		nxge_port_rcr_size;
32253859Sml29623 
32263859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
32276929Smisaki 	    "==> nxge_map_rxdma_channel_cfg_ring"));
32283859Sml29623 
32293859Sml29623 	cntl_dmap = *dma_cntl_p;
32303859Sml29623 
32313859Sml29623 	/* Map in the receive block ring */
32323859Sml29623 	rbrp = *rbr_p;
32333859Sml29623 	dmap = (p_nxge_dma_common_t)&rbrp->rbr_desc;
32343859Sml29623 	nxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4);
32353859Sml29623 	/*
32363859Sml29623 	 * Zero out buffer block ring descriptors.
32373859Sml29623 	 */
32383859Sml29623 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
32393859Sml29623 
32403859Sml29623 	rcfga_p = &(rbrp->rbr_cfga);
32413859Sml29623 	rcfgb_p = &(rbrp->rbr_cfgb);
32423859Sml29623 	kick_p = &(rbrp->rbr_kick);
32433859Sml29623 	rcfga_p->value = 0;
32443859Sml29623 	rcfgb_p->value = 0;
32453859Sml29623 	kick_p->value = 0;
32463859Sml29623 	rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress;
32473859Sml29623 	rcfga_p->value = (rbrp->rbr_addr &
32486929Smisaki 	    (RBR_CFIG_A_STDADDR_MASK |
32496929Smisaki 	    RBR_CFIG_A_STDADDR_BASE_MASK));
32503859Sml29623 	rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT);
32513859Sml29623 
32523859Sml29623 	rcfgb_p->bits.ldw.bufsz0 = rbrp->pkt_buf_size0;
32533859Sml29623 	rcfgb_p->bits.ldw.vld0 = 1;
32543859Sml29623 	rcfgb_p->bits.ldw.bufsz1 = rbrp->pkt_buf_size1;
32553859Sml29623 	rcfgb_p->bits.ldw.vld1 = 1;
32563859Sml29623 	rcfgb_p->bits.ldw.bufsz2 = rbrp->pkt_buf_size2;
32573859Sml29623 	rcfgb_p->bits.ldw.vld2 = 1;
32583859Sml29623 	rcfgb_p->bits.ldw.bksize = nxgep->rx_bksize_code;
32593859Sml29623 
32603859Sml29623 	/*
32613859Sml29623 	 * For each buffer block, enter receive block address to the ring.
32623859Sml29623 	 */
32633859Sml29623 	rbr_vaddrp = (uint32_t *)dmap->kaddrp;
32643859Sml29623 	rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp;
32653859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
32666929Smisaki 	    "==> nxge_map_rxdma_channel_cfg_ring: channel %d "
32676929Smisaki 	    "rbr_vaddrp $%p", dma_channel, rbr_vaddrp));
32683859Sml29623 
32693859Sml29623 	rx_msg_ring = rbrp->rx_msg_ring;
32703859Sml29623 	for (i = 0; i < rbrp->tnblocks; i++) {
32713859Sml29623 		rx_msg_p = rx_msg_ring[i];
32723859Sml29623 		rx_msg_p->nxgep = nxgep;
32733859Sml29623 		rx_msg_p->rx_rbr_p = rbrp;
32743859Sml29623 		bkaddr = (uint32_t)
32756929Smisaki 		    ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress
32766929Smisaki 		    >> RBR_BKADDR_SHIFT));
32773859Sml29623 		rx_msg_p->free = B_FALSE;
32783859Sml29623 		rx_msg_p->max_usage_cnt = 0xbaddcafe;
32793859Sml29623 
32803859Sml29623 		*rbr_vaddrp++ = bkaddr;
32813859Sml29623 	}
32823859Sml29623 
32833859Sml29623 	kick_p->bits.ldw.bkadd = rbrp->rbb_max;
32843859Sml29623 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
32853859Sml29623 
32863859Sml29623 	rbrp->rbr_rd_index = 0;
32873859Sml29623 
32883859Sml29623 	rbrp->rbr_consumed = 0;
32893859Sml29623 	rbrp->rbr_use_bcopy = B_TRUE;
32903859Sml29623 	rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0;
32913859Sml29623 	/*
32923859Sml29623 	 * Do bcopy on packets greater than bcopy size once
32933859Sml29623 	 * the lo threshold is reached.
32943859Sml29623 	 * This lo threshold should be less than the hi threshold.
32953859Sml29623 	 *
32963859Sml29623 	 * Do bcopy on every packet once the hi threshold is reached.
32973859Sml29623 	 */
32983859Sml29623 	if (nxge_rx_threshold_lo >= nxge_rx_threshold_hi) {
32993859Sml29623 		/* default it to use hi */
33003859Sml29623 		nxge_rx_threshold_lo = nxge_rx_threshold_hi;
33013859Sml29623 	}
33023859Sml29623 
33033859Sml29623 	if (nxge_rx_buf_size_type > NXGE_RBR_TYPE2) {
33043859Sml29623 		nxge_rx_buf_size_type = NXGE_RBR_TYPE2;
33053859Sml29623 	}
33063859Sml29623 	rbrp->rbr_bufsize_type = nxge_rx_buf_size_type;
33073859Sml29623 
33083859Sml29623 	switch (nxge_rx_threshold_hi) {
33093859Sml29623 	default:
33103859Sml29623 	case	NXGE_RX_COPY_NONE:
33113859Sml29623 		/* Do not do bcopy at all */
33123859Sml29623 		rbrp->rbr_use_bcopy = B_FALSE;
33133859Sml29623 		rbrp->rbr_threshold_hi = rbrp->rbb_max;
33143859Sml29623 		break;
33153859Sml29623 
33163859Sml29623 	case NXGE_RX_COPY_1:
33173859Sml29623 	case NXGE_RX_COPY_2:
33183859Sml29623 	case NXGE_RX_COPY_3:
33193859Sml29623 	case NXGE_RX_COPY_4:
33203859Sml29623 	case NXGE_RX_COPY_5:
33213859Sml29623 	case NXGE_RX_COPY_6:
33223859Sml29623 	case NXGE_RX_COPY_7:
33233859Sml29623 		rbrp->rbr_threshold_hi =
33246929Smisaki 		    rbrp->rbb_max *
33256929Smisaki 		    (nxge_rx_threshold_hi)/NXGE_RX_BCOPY_SCALE;
33263859Sml29623 		break;
33273859Sml29623 
33283859Sml29623 	case NXGE_RX_COPY_ALL:
33293859Sml29623 		rbrp->rbr_threshold_hi = 0;
33303859Sml29623 		break;
33313859Sml29623 	}
33323859Sml29623 
33333859Sml29623 	switch (nxge_rx_threshold_lo) {
33343859Sml29623 	default:
33353859Sml29623 	case	NXGE_RX_COPY_NONE:
33363859Sml29623 		/* Do not do bcopy at all */
33373859Sml29623 		if (rbrp->rbr_use_bcopy) {
33383859Sml29623 			rbrp->rbr_use_bcopy = B_FALSE;
33393859Sml29623 		}
33403859Sml29623 		rbrp->rbr_threshold_lo = rbrp->rbb_max;
33413859Sml29623 		break;
33423859Sml29623 
33433859Sml29623 	case NXGE_RX_COPY_1:
33443859Sml29623 	case NXGE_RX_COPY_2:
33453859Sml29623 	case NXGE_RX_COPY_3:
33463859Sml29623 	case NXGE_RX_COPY_4:
33473859Sml29623 	case NXGE_RX_COPY_5:
33483859Sml29623 	case NXGE_RX_COPY_6:
33493859Sml29623 	case NXGE_RX_COPY_7:
33503859Sml29623 		rbrp->rbr_threshold_lo =
33516929Smisaki 		    rbrp->rbb_max *
33526929Smisaki 		    (nxge_rx_threshold_lo)/NXGE_RX_BCOPY_SCALE;
33533859Sml29623 		break;
33543859Sml29623 
33553859Sml29623 	case NXGE_RX_COPY_ALL:
33563859Sml29623 		rbrp->rbr_threshold_lo = 0;
33573859Sml29623 		break;
33583859Sml29623 	}
33593859Sml29623 
33603859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
33616929Smisaki 	    "nxge_map_rxdma_channel_cfg_ring: channel %d "
33626929Smisaki 	    "rbb_max %d "
33636929Smisaki 	    "rbrp->rbr_bufsize_type %d "
33646929Smisaki 	    "rbb_threshold_hi %d "
33656929Smisaki 	    "rbb_threshold_lo %d",
33666929Smisaki 	    dma_channel,
33676929Smisaki 	    rbrp->rbb_max,
33686929Smisaki 	    rbrp->rbr_bufsize_type,
33696929Smisaki 	    rbrp->rbr_threshold_hi,
33706929Smisaki 	    rbrp->rbr_threshold_lo));
33713859Sml29623 
33723859Sml29623 	rbrp->page_valid.value = 0;
33733859Sml29623 	rbrp->page_mask_1.value = rbrp->page_mask_2.value = 0;
33743859Sml29623 	rbrp->page_value_1.value = rbrp->page_value_2.value = 0;
33753859Sml29623 	rbrp->page_reloc_1.value = rbrp->page_reloc_2.value = 0;
33763859Sml29623 	rbrp->page_hdl.value = 0;
33773859Sml29623 
33783859Sml29623 	rbrp->page_valid.bits.ldw.page0 = 1;
33793859Sml29623 	rbrp->page_valid.bits.ldw.page1 = 1;
33803859Sml29623 
33813859Sml29623 	/* Map in the receive completion ring */
33823859Sml29623 	rcrp = (p_rx_rcr_ring_t)
33836929Smisaki 	    KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP);
33843859Sml29623 	rcrp->rdc = dma_channel;
33853859Sml29623 
33863859Sml29623 	nxge_port_rcr_size = nxgep->nxge_port_rcr_size;
33873859Sml29623 	rcrp->comp_size = nxge_port_rcr_size;
33883859Sml29623 	rcrp->comp_wrap_mask = nxge_port_rcr_size - 1;
33893859Sml29623 
33903859Sml29623 	rcrp->max_receive_pkts = nxge_max_rx_pkts;
33913859Sml29623 
33923859Sml29623 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
33933859Sml29623 	nxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size,
33946929Smisaki 	    sizeof (rcr_entry_t));
33953859Sml29623 	rcrp->comp_rd_index = 0;
33963859Sml29623 	rcrp->comp_wt_index = 0;
33973859Sml29623 	rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
33986929Smisaki 	    (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
33995125Sjoycey #if defined(__i386)
34006929Smisaki 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
34016929Smisaki 	    (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
34025125Sjoycey #else
34036929Smisaki 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
34046929Smisaki 	    (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
34055125Sjoycey #endif
34063859Sml29623 
34073859Sml29623 	rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
34086929Smisaki 	    (nxge_port_rcr_size - 1);
34093859Sml29623 	rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
34106929Smisaki 	    (nxge_port_rcr_size - 1);
34113859Sml29623 
34123859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
34136929Smisaki 	    "==> nxge_map_rxdma_channel_cfg_ring: "
34146929Smisaki 	    "channel %d "
34156929Smisaki 	    "rbr_vaddrp $%p "
34166929Smisaki 	    "rcr_desc_rd_head_p $%p "
34176929Smisaki 	    "rcr_desc_rd_head_pp $%p "
34186929Smisaki 	    "rcr_desc_rd_last_p $%p "
34196929Smisaki 	    "rcr_desc_rd_last_pp $%p ",
34206929Smisaki 	    dma_channel,
34216929Smisaki 	    rbr_vaddrp,
34226929Smisaki 	    rcrp->rcr_desc_rd_head_p,
34236929Smisaki 	    rcrp->rcr_desc_rd_head_pp,
34246929Smisaki 	    rcrp->rcr_desc_last_p,
34256929Smisaki 	    rcrp->rcr_desc_last_pp));
34263859Sml29623 
34273859Sml29623 	/*
34283859Sml29623 	 * Zero out buffer block ring descriptors.
34293859Sml29623 	 */
34303859Sml29623 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
34313859Sml29623 	rcrp->intr_timeout = nxgep->intr_timeout;
34323859Sml29623 	rcrp->intr_threshold = nxgep->intr_threshold;
34333859Sml29623 	rcrp->full_hdr_flag = B_FALSE;
34343859Sml29623 	rcrp->sw_priv_hdr_len = 0;
34353859Sml29623 
34363859Sml29623 	cfga_p = &(rcrp->rcr_cfga);
34373859Sml29623 	cfgb_p = &(rcrp->rcr_cfgb);
34383859Sml29623 	cfga_p->value = 0;
34393859Sml29623 	cfgb_p->value = 0;
34403859Sml29623 	rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress;
34413859Sml29623 	cfga_p->value = (rcrp->rcr_addr &
34426929Smisaki 	    (RCRCFIG_A_STADDR_MASK |
34436929Smisaki 	    RCRCFIG_A_STADDR_BASE_MASK));
34443859Sml29623 
34453859Sml29623 	rcfga_p->value |= ((uint64_t)rcrp->comp_size <<
34466929Smisaki 	    RCRCFIG_A_LEN_SHIF);
34473859Sml29623 
34483859Sml29623 	/*
34493859Sml29623 	 * Timeout should be set based on the system clock divider.
34503859Sml29623 	 * The following timeout value of 1 assumes that the
34513859Sml29623 	 * granularity (1000) is 3 microseconds running at 300MHz.
34523859Sml29623 	 */
34533859Sml29623 	cfgb_p->bits.ldw.pthres = rcrp->intr_threshold;
34543859Sml29623 	cfgb_p->bits.ldw.timeout = rcrp->intr_timeout;
34553859Sml29623 	cfgb_p->bits.ldw.entout = 1;
34563859Sml29623 
34573859Sml29623 	/* Map in the mailbox */
34583859Sml29623 	mboxp = (p_rx_mbox_t)
34596929Smisaki 	    KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP);
34603859Sml29623 	dmap = (p_nxge_dma_common_t)&mboxp->rx_mbox;
34613859Sml29623 	nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t));
34623859Sml29623 	cfig1_p = (p_rxdma_cfig1_t)&mboxp->rx_cfg1;
34633859Sml29623 	cfig2_p = (p_rxdma_cfig2_t)&mboxp->rx_cfg2;
34643859Sml29623 	cfig1_p->value = cfig2_p->value = 0;
34653859Sml29623 
34663859Sml29623 	mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress;
34673859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
34686929Smisaki 	    "==> nxge_map_rxdma_channel_cfg_ring: "
34696929Smisaki 	    "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx",
34706929Smisaki 	    dma_channel, cfig1_p->value, cfig2_p->value,
34716929Smisaki 	    mboxp->mbox_addr));
34723859Sml29623 
34733859Sml29623 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress >> 32
34746929Smisaki 	    & 0xfff);
34753859Sml29623 	cfig1_p->bits.ldw.mbaddr_h = dmaaddrp;
34763859Sml29623 
34773859Sml29623 
34783859Sml29623 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff);
34793859Sml29623 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress &
34806929Smisaki 	    RXDMA_CFIG2_MBADDR_L_MASK);
34813859Sml29623 
34823859Sml29623 	cfig2_p->bits.ldw.mbaddr = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT);
34833859Sml29623 
34843859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
34856929Smisaki 	    "==> nxge_map_rxdma_channel_cfg_ring: "
34866929Smisaki 	    "channel %d damaddrp $%p "
34876929Smisaki 	    "cfg1 0x%016llx cfig2 0x%016llx",
34886929Smisaki 	    dma_channel, dmaaddrp,
34896929Smisaki 	    cfig1_p->value, cfig2_p->value));
34903859Sml29623 
34913859Sml29623 	cfig2_p->bits.ldw.full_hdr = rcrp->full_hdr_flag;
34923859Sml29623 	cfig2_p->bits.ldw.offset = rcrp->sw_priv_hdr_len;
34933859Sml29623 
34943859Sml29623 	rbrp->rx_rcr_p = rcrp;
34953859Sml29623 	rcrp->rx_rbr_p = rbrp;
34963859Sml29623 	*rcr_p = rcrp;
34973859Sml29623 	*rx_mbox_p = mboxp;
34983859Sml29623 
34993859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
35006929Smisaki 	    "<== nxge_map_rxdma_channel_cfg_ring status 0x%08x", status));
35013859Sml29623 
35023859Sml29623 	return (status);
35033859Sml29623 }
35043859Sml29623 
35053859Sml29623 /*ARGSUSED*/
35063859Sml29623 static void
35073859Sml29623 nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t nxgep,
35083859Sml29623     p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
35093859Sml29623 {
35103859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
35116929Smisaki 	    "==> nxge_unmap_rxdma_channel_cfg_ring: channel %d",
35126929Smisaki 	    rcr_p->rdc));
35133859Sml29623 
35143859Sml29623 	KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t));
35153859Sml29623 	KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t));
35163859Sml29623 
35173859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
35186929Smisaki 	    "<== nxge_unmap_rxdma_channel_cfg_ring"));
35193859Sml29623 }
35203859Sml29623 
35213859Sml29623 static nxge_status_t
35223859Sml29623 nxge_map_rxdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel,
35233859Sml29623     p_nxge_dma_common_t *dma_buf_p,
35243859Sml29623     p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks)
35253859Sml29623 {
35263859Sml29623 	p_rx_rbr_ring_t 	rbrp;
35273859Sml29623 	p_nxge_dma_common_t 	dma_bufp, tmp_bufp;
35283859Sml29623 	p_rx_msg_t 		*rx_msg_ring;
35293859Sml29623 	p_rx_msg_t 		rx_msg_p;
35303859Sml29623 	p_mblk_t 		mblk_p;
35313859Sml29623 
35323859Sml29623 	rxring_info_t *ring_info;
35333859Sml29623 	nxge_status_t		status = NXGE_OK;
35343859Sml29623 	int			i, j, index;
35353859Sml29623 	uint32_t		size, bsize, nblocks, nmsgs;
35363859Sml29623 
35373859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
35386929Smisaki 	    "==> nxge_map_rxdma_channel_buf_ring: channel %d",
35396929Smisaki 	    channel));
35403859Sml29623 
35413859Sml29623 	dma_bufp = tmp_bufp = *dma_buf_p;
35423859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
35436929Smisaki 	    " nxge_map_rxdma_channel_buf_ring: channel %d to map %d "
35446929Smisaki 	    "chunks bufp 0x%016llx",
35456929Smisaki 	    channel, num_chunks, dma_bufp));
35463859Sml29623 
35473859Sml29623 	nmsgs = 0;
35483859Sml29623 	for (i = 0; i < num_chunks; i++, tmp_bufp++) {
35493859Sml29623 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
35506929Smisaki 		    "==> nxge_map_rxdma_channel_buf_ring: channel %d "
35516929Smisaki 		    "bufp 0x%016llx nblocks %d nmsgs %d",
35526929Smisaki 		    channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
35533859Sml29623 		nmsgs += tmp_bufp->nblocks;
35543859Sml29623 	}
35553859Sml29623 	if (!nmsgs) {
35564185Sspeer 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
35576929Smisaki 		    "<== nxge_map_rxdma_channel_buf_ring: channel %d "
35586929Smisaki 		    "no msg blocks",
35596929Smisaki 		    channel));
35603859Sml29623 		status = NXGE_ERROR;
35613859Sml29623 		goto nxge_map_rxdma_channel_buf_ring_exit;
35623859Sml29623 	}
35633859Sml29623 
35645170Stm144005 	rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (*rbrp), KM_SLEEP);
35653859Sml29623 
35663859Sml29623 	size = nmsgs * sizeof (p_rx_msg_t);
35673859Sml29623 	rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
35683859Sml29623 	ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t),
35696929Smisaki 	    KM_SLEEP);
35703859Sml29623 
35713859Sml29623 	MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER,
35726929Smisaki 	    (void *)nxgep->interrupt_cookie);
35733859Sml29623 	MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER,
35746929Smisaki 	    (void *)nxgep->interrupt_cookie);
35753859Sml29623 	rbrp->rdc = channel;
35763859Sml29623 	rbrp->num_blocks = num_chunks;
35773859Sml29623 	rbrp->tnblocks = nmsgs;
35783859Sml29623 	rbrp->rbb_max = nmsgs;
35793859Sml29623 	rbrp->rbr_max_size = nmsgs;
35803859Sml29623 	rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1);
35813859Sml29623 
35823859Sml29623 	/*
35833859Sml29623 	 * Buffer sizes suggested by NIU architect.
35843859Sml29623 	 * 256, 512 and 2K.
35853859Sml29623 	 */
35863859Sml29623 
35873859Sml29623 	rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B;
35883859Sml29623 	rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES;
35893859Sml29623 	rbrp->npi_pkt_buf_size0 = SIZE_256B;
35903859Sml29623 
35913859Sml29623 	rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K;
35923859Sml29623 	rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES;
35933859Sml29623 	rbrp->npi_pkt_buf_size1 = SIZE_1KB;
35943859Sml29623 
35953859Sml29623 	rbrp->block_size = nxgep->rx_default_block_size;
35963859Sml29623 
35973859Sml29623 	if (!nxge_jumbo_enable && !nxgep->param_arr[param_accept_jumbo].value) {
35983859Sml29623 		rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K;
35993859Sml29623 		rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES;
36003859Sml29623 		rbrp->npi_pkt_buf_size2 = SIZE_2KB;
36013859Sml29623 	} else {
36023859Sml29623 		if (rbrp->block_size >= 0x2000) {
36033859Sml29623 			rbrp->pkt_buf_size2 = RBR_BUFSZ2_8K;
36043859Sml29623 			rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_8K_BYTES;
36053859Sml29623 			rbrp->npi_pkt_buf_size2 = SIZE_8KB;
36063859Sml29623 		} else {
36073859Sml29623 			rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K;
36083859Sml29623 			rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES;
36093859Sml29623 			rbrp->npi_pkt_buf_size2 = SIZE_4KB;
36103859Sml29623 		}
36113859Sml29623 	}
36123859Sml29623 
36133859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
36146929Smisaki 	    "==> nxge_map_rxdma_channel_buf_ring: channel %d "
36156929Smisaki 	    "actual rbr max %d rbb_max %d nmsgs %d "
36166929Smisaki 	    "rbrp->block_size %d default_block_size %d "
36176929Smisaki 	    "(config nxge_rbr_size %d nxge_rbr_spare_size %d)",
36186929Smisaki 	    channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs,
36196929Smisaki 	    rbrp->block_size, nxgep->rx_default_block_size,
36206929Smisaki 	    nxge_rbr_size, nxge_rbr_spare_size));
36213859Sml29623 
36223859Sml29623 	/* Map in buffers from the buffer pool.  */
36233859Sml29623 	index = 0;
36243859Sml29623 	for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) {
36253859Sml29623 		bsize = dma_bufp->block_size;
36263859Sml29623 		nblocks = dma_bufp->nblocks;
36275125Sjoycey #if defined(__i386)
36285125Sjoycey 		ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp;
36295125Sjoycey #else
36303859Sml29623 		ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp;
36315125Sjoycey #endif
36323859Sml29623 		ring_info->buffer[i].buf_index = i;
36333859Sml29623 		ring_info->buffer[i].buf_size = dma_bufp->alength;
36343859Sml29623 		ring_info->buffer[i].start_index = index;
36355125Sjoycey #if defined(__i386)
36365125Sjoycey 		ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp;
36375125Sjoycey #else
36383859Sml29623 		ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp;
36395125Sjoycey #endif
36403859Sml29623 
36413859Sml29623 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
36426929Smisaki 		    " nxge_map_rxdma_channel_buf_ring: map channel %d "
36436929Smisaki 		    "chunk %d"
36446929Smisaki 		    " nblocks %d chunk_size %x block_size 0x%x "
36456929Smisaki 		    "dma_bufp $%p", channel, i,
36466929Smisaki 		    dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize,
36476929Smisaki 		    dma_bufp));
36483859Sml29623 
36493859Sml29623 		for (j = 0; j < nblocks; j++) {
36503859Sml29623 			if ((rx_msg_p = nxge_allocb(bsize, BPRI_LO,
36516929Smisaki 			    dma_bufp)) == NULL) {
36524185Sspeer 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
36536929Smisaki 				    "allocb failed (index %d i %d j %d)",
36546929Smisaki 				    index, i, j));
36554185Sspeer 				goto nxge_map_rxdma_channel_buf_ring_fail1;
36563859Sml29623 			}
36573859Sml29623 			rx_msg_ring[index] = rx_msg_p;
36583859Sml29623 			rx_msg_p->block_index = index;
36593859Sml29623 			rx_msg_p->shifted_addr = (uint32_t)
36606929Smisaki 			    ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >>
36616929Smisaki 			    RBR_BKADDR_SHIFT));
36623859Sml29623 
36633859Sml29623 			NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
36646929Smisaki 			    "index %d j %d rx_msg_p $%p mblk %p",
36656929Smisaki 			    index, j, rx_msg_p, rx_msg_p->rx_mblk_p));
36663859Sml29623 
36673859Sml29623 			mblk_p = rx_msg_p->rx_mblk_p;
36683859Sml29623 			mblk_p->b_wptr = mblk_p->b_rptr + bsize;
36695170Stm144005 
36705170Stm144005 			rbrp->rbr_ref_cnt++;
36713859Sml29623 			index++;
36723859Sml29623 			rx_msg_p->buf_dma.dma_channel = channel;
36733859Sml29623 		}
36746495Sspeer 
36756495Sspeer 		rbrp->rbr_alloc_type = DDI_MEM_ALLOC;
36766495Sspeer 		if (dma_bufp->contig_alloc_type) {
36776495Sspeer 			rbrp->rbr_alloc_type = CONTIG_MEM_ALLOC;
36786495Sspeer 		}
36796495Sspeer 
36806495Sspeer 		if (dma_bufp->kmem_alloc_type) {
36816495Sspeer 			rbrp->rbr_alloc_type = KMEM_ALLOC;
36826495Sspeer 		}
36836495Sspeer 
36846495Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
36856495Sspeer 		    " nxge_map_rxdma_channel_buf_ring: map channel %d "
36866495Sspeer 		    "chunk %d"
36876495Sspeer 		    " nblocks %d chunk_size %x block_size 0x%x "
36886495Sspeer 		    "dma_bufp $%p",
36896495Sspeer 		    channel, i,
36906495Sspeer 		    dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize,
36916495Sspeer 		    dma_bufp));
36923859Sml29623 	}
36933859Sml29623 	if (i < rbrp->num_blocks) {
36943859Sml29623 		goto nxge_map_rxdma_channel_buf_ring_fail1;
36953859Sml29623 	}
36963859Sml29623 
36973859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
36986929Smisaki 	    "nxge_map_rxdma_channel_buf_ring: done buf init "
36996929Smisaki 	    "channel %d msg block entries %d",
37006929Smisaki 	    channel, index));
37013859Sml29623 	ring_info->block_size_mask = bsize - 1;
37023859Sml29623 	rbrp->rx_msg_ring = rx_msg_ring;
37033859Sml29623 	rbrp->dma_bufp = dma_buf_p;
37043859Sml29623 	rbrp->ring_info = ring_info;
37053859Sml29623 
37063859Sml29623 	status = nxge_rxbuf_index_info_init(nxgep, rbrp);
37073859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37086929Smisaki 	    " nxge_map_rxdma_channel_buf_ring: "
37096929Smisaki 	    "channel %d done buf info init", channel));
37103859Sml29623 
37115170Stm144005 	/*
37125170Stm144005 	 * Finally, permit nxge_freeb() to call nxge_post_page().
37135170Stm144005 	 */
37145170Stm144005 	rbrp->rbr_state = RBR_POSTING;
37155170Stm144005 
37163859Sml29623 	*rbr_p = rbrp;
37173859Sml29623 	goto nxge_map_rxdma_channel_buf_ring_exit;
37183859Sml29623 
37193859Sml29623 nxge_map_rxdma_channel_buf_ring_fail1:
37203859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37216929Smisaki 	    " nxge_map_rxdma_channel_buf_ring: failed channel (0x%x)",
37226929Smisaki 	    channel, status));
37233859Sml29623 
37243859Sml29623 	index--;
37253859Sml29623 	for (; index >= 0; index--) {
37263859Sml29623 		rx_msg_p = rx_msg_ring[index];
37273859Sml29623 		if (rx_msg_p != NULL) {
37283859Sml29623 			freeb(rx_msg_p->rx_mblk_p);
37293859Sml29623 			rx_msg_ring[index] = NULL;
37303859Sml29623 		}
37313859Sml29623 	}
37323859Sml29623 nxge_map_rxdma_channel_buf_ring_fail:
37333859Sml29623 	MUTEX_DESTROY(&rbrp->post_lock);
37343859Sml29623 	MUTEX_DESTROY(&rbrp->lock);
37353859Sml29623 	KMEM_FREE(ring_info, sizeof (rxring_info_t));
37363859Sml29623 	KMEM_FREE(rx_msg_ring, size);
37373859Sml29623 	KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t));
37383859Sml29623 
37394185Sspeer 	status = NXGE_ERROR;
37404185Sspeer 
37413859Sml29623 nxge_map_rxdma_channel_buf_ring_exit:
37423859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37436929Smisaki 	    "<== nxge_map_rxdma_channel_buf_ring status 0x%08x", status));
37443859Sml29623 
37453859Sml29623 	return (status);
37463859Sml29623 }
37473859Sml29623 
37483859Sml29623 /*ARGSUSED*/
37493859Sml29623 static void
37503859Sml29623 nxge_unmap_rxdma_channel_buf_ring(p_nxge_t nxgep,
37513859Sml29623     p_rx_rbr_ring_t rbr_p)
37523859Sml29623 {
37533859Sml29623 	p_rx_msg_t 		*rx_msg_ring;
37543859Sml29623 	p_rx_msg_t 		rx_msg_p;
37553859Sml29623 	rxring_info_t 		*ring_info;
37563859Sml29623 	int			i;
37573859Sml29623 	uint32_t		size;
37583859Sml29623 #ifdef	NXGE_DEBUG
37593859Sml29623 	int			num_chunks;
37603859Sml29623 #endif
37613859Sml29623 
37623859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37636929Smisaki 	    "==> nxge_unmap_rxdma_channel_buf_ring"));
37643859Sml29623 	if (rbr_p == NULL) {
37653859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
37666929Smisaki 		    "<== nxge_unmap_rxdma_channel_buf_ring: NULL rbrp"));
37673859Sml29623 		return;
37683859Sml29623 	}
37693859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37706929Smisaki 	    "==> nxge_unmap_rxdma_channel_buf_ring: channel %d",
37716929Smisaki 	    rbr_p->rdc));
37723859Sml29623 
37733859Sml29623 	rx_msg_ring = rbr_p->rx_msg_ring;
37743859Sml29623 	ring_info = rbr_p->ring_info;
37753859Sml29623 
37763859Sml29623 	if (rx_msg_ring == NULL || ring_info == NULL) {
37776929Smisaki 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37786929Smisaki 		    "<== nxge_unmap_rxdma_channel_buf_ring: "
37796929Smisaki 		    "rx_msg_ring $%p ring_info $%p",
37806929Smisaki 		    rx_msg_p, ring_info));
37813859Sml29623 		return;
37823859Sml29623 	}
37833859Sml29623 
37843859Sml29623 #ifdef	NXGE_DEBUG
37853859Sml29623 	num_chunks = rbr_p->num_blocks;
37863859Sml29623 #endif
37873859Sml29623 	size = rbr_p->tnblocks * sizeof (p_rx_msg_t);
37883859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37896929Smisaki 	    " nxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d "
37906929Smisaki 	    "tnblocks %d (max %d) size ptrs %d ",
37916929Smisaki 	    rbr_p->rdc, num_chunks,
37926929Smisaki 	    rbr_p->tnblocks, rbr_p->rbr_max_size, size));
37933859Sml29623 
37943859Sml29623 	for (i = 0; i < rbr_p->tnblocks; i++) {
37953859Sml29623 		rx_msg_p = rx_msg_ring[i];
37963859Sml29623 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
37976929Smisaki 		    " nxge_unmap_rxdma_channel_buf_ring: "
37986929Smisaki 		    "rx_msg_p $%p",
37996929Smisaki 		    rx_msg_p));
38003859Sml29623 		if (rx_msg_p != NULL) {
38013859Sml29623 			freeb(rx_msg_p->rx_mblk_p);
38023859Sml29623 			rx_msg_ring[i] = NULL;
38033859Sml29623 		}
38043859Sml29623 	}
38053859Sml29623 
38065170Stm144005 	/*
38075170Stm144005 	 * We no longer may use the mutex <post_lock>. By setting
38085170Stm144005 	 * <rbr_state> to anything but POSTING, we prevent
38095170Stm144005 	 * nxge_post_page() from accessing a dead mutex.
38105170Stm144005 	 */
38115170Stm144005 	rbr_p->rbr_state = RBR_UNMAPPING;
38123859Sml29623 	MUTEX_DESTROY(&rbr_p->post_lock);
38135170Stm144005 
38143859Sml29623 	MUTEX_DESTROY(&rbr_p->lock);
38155170Stm144005 
38165170Stm144005 	if (rbr_p->rbr_ref_cnt == 0) {
38176495Sspeer 		/*
38186495Sspeer 		 * This is the normal state of affairs.
38196495Sspeer 		 * Need to free the following buffers:
38206495Sspeer 		 *  - data buffers
38216495Sspeer 		 *  - rx_msg ring
38226495Sspeer 		 *  - ring_info
38236495Sspeer 		 *  - rbr ring
38246495Sspeer 		 */
38256495Sspeer 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
38266495Sspeer 		    "unmap_rxdma_buf_ring: No outstanding - freeing "));
38276495Sspeer 		nxge_rxdma_databuf_free(rbr_p);
38286495Sspeer 		KMEM_FREE(ring_info, sizeof (rxring_info_t));
38296495Sspeer 		KMEM_FREE(rx_msg_ring, size);
38305170Stm144005 		KMEM_FREE(rbr_p, sizeof (*rbr_p));
38315170Stm144005 	} else {
38325170Stm144005 		/*
38335170Stm144005 		 * Some of our buffers are still being used.
38345170Stm144005 		 * Therefore, tell nxge_freeb() this ring is
38355170Stm144005 		 * unmapped, so it may free <rbr_p> for us.
38365170Stm144005 		 */
38375170Stm144005 		rbr_p->rbr_state = RBR_UNMAPPED;
38385170Stm144005 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
38395170Stm144005 		    "unmap_rxdma_buf_ring: %d %s outstanding.",
38405170Stm144005 		    rbr_p->rbr_ref_cnt,
38415170Stm144005 		    rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs"));
38425170Stm144005 	}
38433859Sml29623 
38443859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
38456929Smisaki 	    "<== nxge_unmap_rxdma_channel_buf_ring"));
38463859Sml29623 }
38473859Sml29623 
38486495Sspeer /*
38496495Sspeer  * nxge_rxdma_hw_start_common
38506495Sspeer  *
38516495Sspeer  * Arguments:
38526495Sspeer  * 	nxgep
38536495Sspeer  *
38546495Sspeer  * Notes:
38556495Sspeer  *
38566495Sspeer  * NPI/NXGE function calls:
38576495Sspeer  *	nxge_init_fzc_rx_common();
38586495Sspeer  *	nxge_init_fzc_rxdma_port();
38596495Sspeer  *
38606495Sspeer  * Registers accessed:
38616495Sspeer  *
38626495Sspeer  * Context:
38636495Sspeer  *	Service domain
38646495Sspeer  */
38653859Sml29623 static nxge_status_t
38663859Sml29623 nxge_rxdma_hw_start_common(p_nxge_t nxgep)
38673859Sml29623 {
38683859Sml29623 	nxge_status_t		status = NXGE_OK;
38693859Sml29623 
38703859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common"));
38713859Sml29623 
38723859Sml29623 	/*
38733859Sml29623 	 * Load the sharable parameters by writing to the
38743859Sml29623 	 * function zero control registers. These FZC registers
38753859Sml29623 	 * should be initialized only once for the entire chip.
38763859Sml29623 	 */
38773859Sml29623 	(void) nxge_init_fzc_rx_common(nxgep);
38783859Sml29623 
38793859Sml29623 	/*
38803859Sml29623 	 * Initialize the RXDMA port specific FZC control configurations.
38813859Sml29623 	 * These FZC registers are pertaining to each port.
38823859Sml29623 	 */
38833859Sml29623 	(void) nxge_init_fzc_rxdma_port(nxgep);
38843859Sml29623 
38853859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common"));
38863859Sml29623 
38873859Sml29623 	return (status);
38883859Sml29623 }
38893859Sml29623 
38903859Sml29623 static nxge_status_t
38916495Sspeer nxge_rxdma_hw_start(p_nxge_t nxgep, int channel)
38923859Sml29623 {
38933859Sml29623 	int			i, ndmas;
38943859Sml29623 	p_rx_rbr_rings_t 	rx_rbr_rings;
38953859Sml29623 	p_rx_rbr_ring_t		*rbr_rings;
38963859Sml29623 	p_rx_rcr_rings_t 	rx_rcr_rings;
38973859Sml29623 	p_rx_rcr_ring_t		*rcr_rings;
38983859Sml29623 	p_rx_mbox_areas_t 	rx_mbox_areas_p;
38993859Sml29623 	p_rx_mbox_t		*rx_mbox_p;
39003859Sml29623 	nxge_status_t		status = NXGE_OK;
39013859Sml29623 
39023859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start"));
39033859Sml29623 
39043859Sml29623 	rx_rbr_rings = nxgep->rx_rbr_rings;
39053859Sml29623 	rx_rcr_rings = nxgep->rx_rcr_rings;
39063859Sml29623 	if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
39073859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
39086929Smisaki 		    "<== nxge_rxdma_hw_start: NULL ring pointers"));
39093859Sml29623 		return (NXGE_ERROR);
39103859Sml29623 	}
39113859Sml29623 	ndmas = rx_rbr_rings->ndmas;
39123859Sml29623 	if (ndmas == 0) {
39133859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
39146929Smisaki 		    "<== nxge_rxdma_hw_start: no dma channel allocated"));
39153859Sml29623 		return (NXGE_ERROR);
39163859Sml29623 	}
39173859Sml29623 
39183859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
39196929Smisaki 	    "==> nxge_rxdma_hw_start (ndmas %d)", ndmas));
39203859Sml29623 
39213859Sml29623 	rbr_rings = rx_rbr_rings->rbr_rings;
39223859Sml29623 	rcr_rings = rx_rcr_rings->rcr_rings;
39233859Sml29623 	rx_mbox_areas_p = nxgep->rx_mbox_areas_p;
39243859Sml29623 	if (rx_mbox_areas_p) {
39253859Sml29623 		rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
39263859Sml29623 	}
39273859Sml29623 
39286495Sspeer 	i = channel;
39296495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
39306929Smisaki 	    "==> nxge_rxdma_hw_start (ndmas %d) channel %d",
39316929Smisaki 	    ndmas, channel));
39326495Sspeer 	status = nxge_rxdma_start_channel(nxgep, channel,
39336495Sspeer 	    (p_rx_rbr_ring_t)rbr_rings[i],
39346495Sspeer 	    (p_rx_rcr_ring_t)rcr_rings[i],
39356495Sspeer 	    (p_rx_mbox_t)rx_mbox_p[i]);
39366495Sspeer 	if (status != NXGE_OK) {
39376495Sspeer 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
39386495Sspeer 		    "==> nxge_rxdma_hw_start: disable "
39396495Sspeer 		    "(status 0x%x channel %d)", status, channel));
39406495Sspeer 		return (status);
39413859Sml29623 	}
39423859Sml29623 
39433859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start: "
39446929Smisaki 	    "rx_rbr_rings 0x%016llx rings 0x%016llx",
39456929Smisaki 	    rx_rbr_rings, rx_rcr_rings));
39463859Sml29623 
39473859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
39486929Smisaki 	    "==> nxge_rxdma_hw_start: (status 0x%x)", status));
39493859Sml29623 
39503859Sml29623 	return (status);
39513859Sml29623 }
39523859Sml29623 
39533859Sml29623 static void
39546495Sspeer nxge_rxdma_hw_stop(p_nxge_t nxgep, int channel)
39553859Sml29623 {
39563859Sml29623 	p_rx_rbr_rings_t 	rx_rbr_rings;
39573859Sml29623 	p_rx_rcr_rings_t 	rx_rcr_rings;
39583859Sml29623 
39593859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop"));
39603859Sml29623 
39613859Sml29623 	rx_rbr_rings = nxgep->rx_rbr_rings;
39623859Sml29623 	rx_rcr_rings = nxgep->rx_rcr_rings;
39633859Sml29623 	if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
39643859Sml29623 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
39656929Smisaki 		    "<== nxge_rxdma_hw_stop: NULL ring pointers"));
39663859Sml29623 		return;
39673859Sml29623 	}
39683859Sml29623 
39693859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
39706929Smisaki 	    "==> nxge_rxdma_hw_stop(channel %d)",
39716929Smisaki 	    channel));
39726495Sspeer 	(void) nxge_rxdma_stop_channel(nxgep, channel);
39733859Sml29623 
39743859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop: "
39756929Smisaki 	    "rx_rbr_rings 0x%016llx rings 0x%016llx",
39766929Smisaki 	    rx_rbr_rings, rx_rcr_rings));
39773859Sml29623 
39783859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_hw_stop"));
39793859Sml29623 }
39803859Sml29623 
39813859Sml29623 
39823859Sml29623 static nxge_status_t
39833859Sml29623 nxge_rxdma_start_channel(p_nxge_t nxgep, uint16_t channel,
39843859Sml29623     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p)
39853859Sml29623 
39863859Sml29623 {
39873859Sml29623 	npi_handle_t		handle;
39883859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
39893859Sml29623 	rx_dma_ctl_stat_t	cs;
39903859Sml29623 	rx_dma_ent_msk_t	ent_mask;
39913859Sml29623 	nxge_status_t		status = NXGE_OK;
39923859Sml29623 
39933859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel"));
39943859Sml29623 
39953859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
39963859Sml29623 
39973859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "nxge_rxdma_start_channel: "
39983859Sml29623 		"npi handle addr $%p acc $%p",
39993859Sml29623 		nxgep->npi_handle.regp, nxgep->npi_handle.regh));
40003859Sml29623 
40016495Sspeer 	/* Reset RXDMA channel, but not if you're a guest. */
40026495Sspeer 	if (!isLDOMguest(nxgep)) {
40036495Sspeer 		rs = npi_rxdma_cfg_rdc_reset(handle, channel);
40046495Sspeer 		if (rs != NPI_SUCCESS) {
40056495Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
40066495Sspeer 			    "==> nxge_init_fzc_rdc: "
40076495Sspeer 			    "npi_rxdma_cfg_rdc_reset(%d) returned 0x%08x",
40086495Sspeer 			    channel, rs));
40096495Sspeer 			return (NXGE_ERROR | rs);
40106495Sspeer 		}
40116495Sspeer 
40126495Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
40136495Sspeer 		    "==> nxge_rxdma_start_channel: reset done: channel %d",
40146495Sspeer 		    channel));
40153859Sml29623 	}
40163859Sml29623 
40176495Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
40186495Sspeer 	if (isLDOMguest(nxgep))
40196495Sspeer 		(void) nxge_rdc_lp_conf(nxgep, channel);
40206495Sspeer #endif
40213859Sml29623 
40223859Sml29623 	/*
40233859Sml29623 	 * Initialize the RXDMA channel specific FZC control
40243859Sml29623 	 * configurations. These FZC registers are pertaining
40253859Sml29623 	 * to each RX channel (logical pages).
40263859Sml29623 	 */
40276495Sspeer 	if (!isLDOMguest(nxgep)) {
40286495Sspeer 		status = nxge_init_fzc_rxdma_channel(nxgep, channel);
40296495Sspeer 		if (status != NXGE_OK) {
40306495Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
40316495Sspeer 				"==> nxge_rxdma_start_channel: "
40326495Sspeer 				"init fzc rxdma failed (0x%08x channel %d)",
40336495Sspeer 				status, channel));
40346495Sspeer 			return (status);
40356495Sspeer 		}
40366495Sspeer 
40376495Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
40386495Sspeer 			"==> nxge_rxdma_start_channel: fzc done"));
40393859Sml29623 	}
40403859Sml29623 
40413859Sml29623 	/* Set up the interrupt event masks. */
40423859Sml29623 	ent_mask.value = 0;
40433859Sml29623 	ent_mask.value |= RX_DMA_ENT_MSK_RBREMPTY_MASK;
40443859Sml29623 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
40456495Sspeer 	    &ent_mask);
40463859Sml29623 	if (rs != NPI_SUCCESS) {
40473859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
40483859Sml29623 			"==> nxge_rxdma_start_channel: "
40496495Sspeer 			"init rxdma event masks failed "
40506495Sspeer 			"(0x%08x channel %d)",
40513859Sml29623 			status, channel));
40523859Sml29623 		return (NXGE_ERROR | rs);
40533859Sml29623 	}
40543859Sml29623 
40556495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
40566495Sspeer 		"==> nxge_rxdma_start_channel: "
40573859Sml29623 		"event done: channel %d (mask 0x%016llx)",
40583859Sml29623 		channel, ent_mask.value));
40593859Sml29623 
40603859Sml29623 	/* Initialize the receive DMA control and status register */
40613859Sml29623 	cs.value = 0;
40623859Sml29623 	cs.bits.hdw.mex = 1;
40633859Sml29623 	cs.bits.hdw.rcrthres = 1;
40643859Sml29623 	cs.bits.hdw.rcrto = 1;
40653859Sml29623 	cs.bits.hdw.rbr_empty = 1;
40663859Sml29623 	status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs);
40673859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
40683859Sml29623 		"channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value));
40693859Sml29623 	if (status != NXGE_OK) {
40703859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
40713859Sml29623 			"==> nxge_rxdma_start_channel: "
40723859Sml29623 			"init rxdma control register failed (0x%08x channel %d",
40733859Sml29623 			status, channel));
40743859Sml29623 		return (status);
40753859Sml29623 	}
40763859Sml29623 
40773859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
40783859Sml29623 		"control done - channel %d cs 0x%016llx", channel, cs.value));
40793859Sml29623 
40803859Sml29623 	/*
40813859Sml29623 	 * Load RXDMA descriptors, buffers, mailbox,
40823859Sml29623 	 * initialise the receive DMA channels and
40833859Sml29623 	 * enable each DMA channel.
40843859Sml29623 	 */
40853859Sml29623 	status = nxge_enable_rxdma_channel(nxgep,
40866495Sspeer 	    channel, rbr_p, rcr_p, mbox_p);
40873859Sml29623 
40883859Sml29623 	if (status != NXGE_OK) {
40893859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
40906495Sspeer 		    " nxge_rxdma_start_channel: "
40916495Sspeer 		    " enable rxdma failed (0x%08x channel %d)",
40926495Sspeer 		    status, channel));
40936495Sspeer 		return (status);
40946495Sspeer 	}
40956495Sspeer 
40966495Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
40976495Sspeer 	    "==> nxge_rxdma_start_channel: enabled channel %d"));
40986495Sspeer 
40996495Sspeer 	if (isLDOMguest(nxgep)) {
41006495Sspeer 		/* Add interrupt handler for this channel. */
41016495Sspeer 		if (nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel)
41026495Sspeer 		    != NXGE_OK) {
41036495Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
41043859Sml29623 			    " nxge_rxdma_start_channel: "
41056495Sspeer 			    " nxge_hio_intr_add failed (0x%08x channel %d)",
41066495Sspeer 		    status, channel));
41076495Sspeer 		}
41083859Sml29623 	}
41093859Sml29623 
41103859Sml29623 	ent_mask.value = 0;
41113859Sml29623 	ent_mask.value |= (RX_DMA_ENT_MSK_WRED_DROP_MASK |
41123859Sml29623 				RX_DMA_ENT_MSK_PTDROP_PKT_MASK);
41133859Sml29623 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
41143859Sml29623 			&ent_mask);
41153859Sml29623 	if (rs != NPI_SUCCESS) {
41163859Sml29623 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
41173859Sml29623 			"==> nxge_rxdma_start_channel: "
41183859Sml29623 			"init rxdma event masks failed (0x%08x channel %d)",
41193859Sml29623 			status, channel));
41203859Sml29623 		return (NXGE_ERROR | rs);
41213859Sml29623 	}
41223859Sml29623 
41233859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
41243859Sml29623 		"control done - channel %d cs 0x%016llx", channel, cs.value));
41253859Sml29623 
41263859Sml29623 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_start_channel"));
41273859Sml29623 
41283859Sml29623 	return (NXGE_OK);
41293859Sml29623 }
41303859Sml29623 
41313859Sml29623 static nxge_status_t
41323859Sml29623 nxge_rxdma_stop_channel(p_nxge_t nxgep, uint16_t channel)
41333859Sml29623 {
41343859Sml29623 	npi_handle_t		handle;
41353859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
41363859Sml29623 	rx_dma_ctl_stat_t	cs;
41373859Sml29623 	rx_dma_ent_msk_t	ent_mask;
41383859Sml29623 	nxge_status_t		status = NXGE_OK;
41393859Sml29623 
41403859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel"));
41413859Sml29623 
41423859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
41433859Sml29623 
41443859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "nxge_rxdma_stop_channel: "
41456929Smisaki 	    "npi handle addr $%p acc $%p",
41466929Smisaki 	    nxgep->npi_handle.regp, nxgep->npi_handle.regh));
41473859Sml29623 
4148*7812SMichael.Speer@Sun.COM 	if (!isLDOMguest(nxgep)) {
4149*7812SMichael.Speer@Sun.COM 		/*
4150*7812SMichael.Speer@Sun.COM 		 * Stop RxMAC = A.9.2.6
4151*7812SMichael.Speer@Sun.COM 		 */
4152*7812SMichael.Speer@Sun.COM 		if (nxge_rx_mac_disable(nxgep) != NXGE_OK) {
4153*7812SMichael.Speer@Sun.COM 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4154*7812SMichael.Speer@Sun.COM 			    "nxge_rxdma_stop_channel: "
4155*7812SMichael.Speer@Sun.COM 			    "Failed to disable RxMAC"));
4156*7812SMichael.Speer@Sun.COM 		}
4157*7812SMichael.Speer@Sun.COM 
4158*7812SMichael.Speer@Sun.COM 		/*
4159*7812SMichael.Speer@Sun.COM 		 * Drain IPP Port = A.9.3.6
4160*7812SMichael.Speer@Sun.COM 		 */
4161*7812SMichael.Speer@Sun.COM 		(void) nxge_ipp_drain(nxgep);
4162*7812SMichael.Speer@Sun.COM 	}
4163*7812SMichael.Speer@Sun.COM 
41643859Sml29623 	/* Reset RXDMA channel */
41653859Sml29623 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
41663859Sml29623 	if (rs != NPI_SUCCESS) {
41673859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
41686929Smisaki 		    " nxge_rxdma_stop_channel: "
41696929Smisaki 		    " reset rxdma failed (0x%08x channel %d)",
41706929Smisaki 		    rs, channel));
41713859Sml29623 		return (NXGE_ERROR | rs);
41723859Sml29623 	}
41733859Sml29623 
41743859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
41756929Smisaki 	    "==> nxge_rxdma_stop_channel: reset done"));
41763859Sml29623 
41773859Sml29623 	/* Set up the interrupt event masks. */
41783859Sml29623 	ent_mask.value = RX_DMA_ENT_MSK_ALL;
41793859Sml29623 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
41806929Smisaki 	    &ent_mask);
41813859Sml29623 	if (rs != NPI_SUCCESS) {
41823859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
41836929Smisaki 		    "==> nxge_rxdma_stop_channel: "
41846929Smisaki 		    "set rxdma event masks failed (0x%08x channel %d)",
41856929Smisaki 		    rs, channel));
41863859Sml29623 		return (NXGE_ERROR | rs);
41873859Sml29623 	}
41883859Sml29623 
41893859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
41906929Smisaki 	    "==> nxge_rxdma_stop_channel: event done"));
41913859Sml29623 
4192*7812SMichael.Speer@Sun.COM 	/*
4193*7812SMichael.Speer@Sun.COM 	 * Initialize the receive DMA control and status register
4194*7812SMichael.Speer@Sun.COM 	 */
41953859Sml29623 	cs.value = 0;
4196*7812SMichael.Speer@Sun.COM 	status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs);
41973859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel: control "
41986929Smisaki 	    " to default (all 0s) 0x%08x", cs.value));
41993859Sml29623 	if (status != NXGE_OK) {
42003859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
42016929Smisaki 		    " nxge_rxdma_stop_channel: init rxdma"
42026929Smisaki 		    " control register failed (0x%08x channel %d",
42036929Smisaki 		    status, channel));
42043859Sml29623 		return (status);
42053859Sml29623 	}
42063859Sml29623 
42073859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
42086929Smisaki 	    "==> nxge_rxdma_stop_channel: control done"));
42093859Sml29623 
4210*7812SMichael.Speer@Sun.COM 	/*
4211*7812SMichael.Speer@Sun.COM 	 * Make sure channel is disabled.
4212*7812SMichael.Speer@Sun.COM 	 */
42133859Sml29623 	status = nxge_disable_rxdma_channel(nxgep, channel);
42143859Sml29623 	if (status != NXGE_OK) {
42153859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
42166929Smisaki 		    " nxge_rxdma_stop_channel: "
42176929Smisaki 		    " init enable rxdma failed (0x%08x channel %d)",
42186929Smisaki 		    status, channel));
42193859Sml29623 		return (status);
42203859Sml29623 	}
42213859Sml29623 
4222*7812SMichael.Speer@Sun.COM 	if (!isLDOMguest(nxgep)) {
4223*7812SMichael.Speer@Sun.COM 		/*
4224*7812SMichael.Speer@Sun.COM 		 * Enable RxMAC = A.9.2.10
4225*7812SMichael.Speer@Sun.COM 		 */
4226*7812SMichael.Speer@Sun.COM 		if (nxge_rx_mac_enable(nxgep) != NXGE_OK) {
4227*7812SMichael.Speer@Sun.COM 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4228*7812SMichael.Speer@Sun.COM 			    "nxge_rxdma_stop_channel: Rx MAC still disabled"));
4229*7812SMichael.Speer@Sun.COM 		}
4230*7812SMichael.Speer@Sun.COM 	}
4231*7812SMichael.Speer@Sun.COM 
42323859Sml29623 	NXGE_DEBUG_MSG((nxgep,
42336929Smisaki 	    RX_CTL, "==> nxge_rxdma_stop_channel: disable done"));
42343859Sml29623 
42353859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_channel"));
42363859Sml29623 
42373859Sml29623 	return (NXGE_OK);
42383859Sml29623 }
42393859Sml29623 
42403859Sml29623 nxge_status_t
42413859Sml29623 nxge_rxdma_handle_sys_errors(p_nxge_t nxgep)
42423859Sml29623 {
42433859Sml29623 	npi_handle_t		handle;
42443859Sml29623 	p_nxge_rdc_sys_stats_t	statsp;
42453859Sml29623 	rx_ctl_dat_fifo_stat_t	stat;
42463859Sml29623 	uint32_t		zcp_err_status;
42473859Sml29623 	uint32_t		ipp_err_status;
42483859Sml29623 	nxge_status_t		status = NXGE_OK;
42493859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
42503859Sml29623 	boolean_t		my_err = B_FALSE;
42513859Sml29623 
42523859Sml29623 	handle = nxgep->npi_handle;
42533859Sml29623 	statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats;
42543859Sml29623 
42553859Sml29623 	rs = npi_rxdma_rxctl_fifo_error_intr_get(handle, &stat);
42563859Sml29623 
42573859Sml29623 	if (rs != NPI_SUCCESS)
42583859Sml29623 		return (NXGE_ERROR | rs);
42593859Sml29623 
42603859Sml29623 	if (stat.bits.ldw.id_mismatch) {
42613859Sml29623 		statsp->id_mismatch++;
42623859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL,
42636929Smisaki 		    NXGE_FM_EREPORT_RDMC_ID_MISMATCH);
42643859Sml29623 		/* Global fatal error encountered */
42653859Sml29623 	}
42663859Sml29623 
42673859Sml29623 	if ((stat.bits.ldw.zcp_eop_err) || (stat.bits.ldw.ipp_eop_err)) {
42683859Sml29623 		switch (nxgep->mac.portnum) {
42693859Sml29623 		case 0:
42703859Sml29623 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT0) ||
42716929Smisaki 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT0)) {
42723859Sml29623 				my_err = B_TRUE;
42733859Sml29623 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
42743859Sml29623 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
42753859Sml29623 			}
42763859Sml29623 			break;
42773859Sml29623 		case 1:
42783859Sml29623 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT1) ||
42796929Smisaki 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT1)) {
42803859Sml29623 				my_err = B_TRUE;
42813859Sml29623 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
42823859Sml29623 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
42833859Sml29623 			}
42843859Sml29623 			break;
42853859Sml29623 		case 2:
42863859Sml29623 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT2) ||
42876929Smisaki 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT2)) {
42883859Sml29623 				my_err = B_TRUE;
42893859Sml29623 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
42903859Sml29623 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
42913859Sml29623 			}
42923859Sml29623 			break;
42933859Sml29623 		case 3:
42943859Sml29623 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT3) ||
42956929Smisaki 			    (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT3)) {
42963859Sml29623 				my_err = B_TRUE;
42973859Sml29623 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
42983859Sml29623 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
42993859Sml29623 			}
43003859Sml29623 			break;
43013859Sml29623 		default:
43023859Sml29623 			return (NXGE_ERROR);
43033859Sml29623 		}
43043859Sml29623 	}
43053859Sml29623 
43063859Sml29623 	if (my_err) {
43073859Sml29623 		status = nxge_rxdma_handle_port_errors(nxgep, ipp_err_status,
43086929Smisaki 		    zcp_err_status);
43093859Sml29623 		if (status != NXGE_OK)
43103859Sml29623 			return (status);
43113859Sml29623 	}
43123859Sml29623 
43133859Sml29623 	return (NXGE_OK);
43143859Sml29623 }
43153859Sml29623 
43163859Sml29623 static nxge_status_t
43173859Sml29623 nxge_rxdma_handle_port_errors(p_nxge_t nxgep, uint32_t ipp_status,
43183859Sml29623 							uint32_t zcp_status)
43193859Sml29623 {
43203859Sml29623 	boolean_t		rxport_fatal = B_FALSE;
43213859Sml29623 	p_nxge_rdc_sys_stats_t	statsp;
43223859Sml29623 	nxge_status_t		status = NXGE_OK;
43233859Sml29623 	uint8_t			portn;
43243859Sml29623 
43253859Sml29623 	portn = nxgep->mac.portnum;
43263859Sml29623 	statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats;
43273859Sml29623 
43283859Sml29623 	if (ipp_status & (0x1 << portn)) {
43293859Sml29623 		statsp->ipp_eop_err++;
43303859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, NULL,
43316929Smisaki 		    NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR);
43323859Sml29623 		rxport_fatal = B_TRUE;
43333859Sml29623 	}
43343859Sml29623 
43353859Sml29623 	if (zcp_status & (0x1 << portn)) {
43363859Sml29623 		statsp->zcp_eop_err++;
43373859Sml29623 		NXGE_FM_REPORT_ERROR(nxgep, portn, NULL,
43386929Smisaki 		    NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR);
43393859Sml29623 		rxport_fatal = B_TRUE;
43403859Sml29623 	}
43413859Sml29623 
43423859Sml29623 	if (rxport_fatal) {
43433859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
43446929Smisaki 		    " nxge_rxdma_handle_port_error: "
43456929Smisaki 		    " fatal error on Port #%d\n",
43466929Smisaki 		    portn));
43473859Sml29623 		status = nxge_rx_port_fatal_err_recover(nxgep);
43483859Sml29623 		if (status == NXGE_OK) {
43493859Sml29623 			FM_SERVICE_RESTORED(nxgep);
43503859Sml29623 		}
43513859Sml29623 	}
43523859Sml29623 
43533859Sml29623 	return (status);
43543859Sml29623 }
43553859Sml29623 
43563859Sml29623 static nxge_status_t
43573859Sml29623 nxge_rxdma_fatal_err_recover(p_nxge_t nxgep, uint16_t channel)
43583859Sml29623 {
43593859Sml29623 	npi_handle_t		handle;
43603859Sml29623 	npi_status_t		rs = NPI_SUCCESS;
43613859Sml29623 	nxge_status_t		status = NXGE_OK;
43623859Sml29623 	p_rx_rbr_ring_t		rbrp;
43633859Sml29623 	p_rx_rcr_ring_t		rcrp;
43643859Sml29623 	p_rx_mbox_t		mboxp;
43653859Sml29623 	rx_dma_ent_msk_t	ent_mask;
43663859Sml29623 	p_nxge_dma_common_t	dmap;
43673859Sml29623 	int			ring_idx;
43683859Sml29623 	uint32_t		ref_cnt;
43693859Sml29623 	p_rx_msg_t		rx_msg_p;
43703859Sml29623 	int			i;
43713859Sml29623 	uint32_t		nxge_port_rcr_size;
43723859Sml29623 
43733859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fatal_err_recover"));
43743859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
43756929Smisaki 	    "Recovering from RxDMAChannel#%d error...", channel));
43763859Sml29623 
43773859Sml29623 	/*
43783859Sml29623 	 * Stop the dma channel waits for the stop done.
43793859Sml29623 	 * If the stop done bit is not set, then create
43803859Sml29623 	 * an error.
43813859Sml29623 	 */
43823859Sml29623 
43833859Sml29623 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
43843859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Rx DMA stop..."));
43853859Sml29623 
43863859Sml29623 	ring_idx = nxge_rxdma_get_ring_index(nxgep, channel);
43873859Sml29623 	rbrp = (p_rx_rbr_ring_t)nxgep->rx_rbr_rings->rbr_rings[ring_idx];
43883859Sml29623 	rcrp = (p_rx_rcr_ring_t)nxgep->rx_rcr_rings->rcr_rings[ring_idx];
43893859Sml29623 
43903859Sml29623 	MUTEX_ENTER(&rcrp->lock);
43913859Sml29623 	MUTEX_ENTER(&rbrp->lock);
43923859Sml29623 	MUTEX_ENTER(&rbrp->post_lock);
43933859Sml29623 
43943859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA channel..."));
43953859Sml29623 
43963859Sml29623 	rs = npi_rxdma_cfg_rdc_disable(handle, channel);
43973859Sml29623 	if (rs != NPI_SUCCESS) {
43983859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
43996929Smisaki 		    "nxge_disable_rxdma_channel:failed"));
44003859Sml29623 		goto fail;
44013859Sml29623 	}
44023859Sml29623 
44033859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA interrupt..."));
44043859Sml29623 
44053859Sml29623 	/* Disable interrupt */
44063859Sml29623 	ent_mask.value = RX_DMA_ENT_MSK_ALL;
44073859Sml29623 	rs = npi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
44083859Sml29623 	if (rs != NPI_SUCCESS) {
44093859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
44106929Smisaki 		    "nxge_rxdma_stop_channel: "
44116929Smisaki 		    "set rxdma event masks failed (channel %d)",
44126929Smisaki 		    channel));
44133859Sml29623 	}
44143859Sml29623 
44153859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel reset..."));
44163859Sml29623 
44173859Sml29623 	/* Reset RXDMA channel */
44183859Sml29623 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
44193859Sml29623 	if (rs != NPI_SUCCESS) {
44203859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
44216929Smisaki 		    "nxge_rxdma_fatal_err_recover: "
44226929Smisaki 		    " reset rxdma failed (channel %d)", channel));
44233859Sml29623 		goto fail;
44243859Sml29623 	}
44253859Sml29623 
44263859Sml29623 	nxge_port_rcr_size = nxgep->nxge_port_rcr_size;
44273859Sml29623 
44283859Sml29623 	mboxp =
44296929Smisaki 	    (p_rx_mbox_t)nxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx];
44303859Sml29623 
44313859Sml29623 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
44323859Sml29623 	rbrp->rbr_rd_index = 0;
44333859Sml29623 
44343859Sml29623 	rcrp->comp_rd_index = 0;
44353859Sml29623 	rcrp->comp_wt_index = 0;
44363859Sml29623 	rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
44376929Smisaki 	    (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
44385125Sjoycey #if defined(__i386)
44396929Smisaki 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
44406929Smisaki 	    (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
44415125Sjoycey #else
44426929Smisaki 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
44436929Smisaki 	    (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
44445125Sjoycey #endif
44453859Sml29623 
44463859Sml29623 	rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
44476929Smisaki 	    (nxge_port_rcr_size - 1);
44483859Sml29623 	rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
44496929Smisaki 	    (nxge_port_rcr_size - 1);
44503859Sml29623 
44513859Sml29623 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
44523859Sml29623 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
44533859Sml29623 
44543859Sml29623 	cmn_err(CE_NOTE, "!rbr entries = %d\n", rbrp->rbr_max_size);
44553859Sml29623 
44563859Sml29623 	for (i = 0; i < rbrp->rbr_max_size; i++) {
44573859Sml29623 		rx_msg_p = rbrp->rx_msg_ring[i];
44583859Sml29623 		ref_cnt = rx_msg_p->ref_cnt;
44593859Sml29623 		if (ref_cnt != 1) {
44603859Sml29623 			if (rx_msg_p->cur_usage_cnt !=
44616929Smisaki 			    rx_msg_p->max_usage_cnt) {
44623859Sml29623 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
44636929Smisaki 				    "buf[%d]: cur_usage_cnt = %d "
44646929Smisaki 				    "max_usage_cnt = %d\n", i,
44656929Smisaki 				    rx_msg_p->cur_usage_cnt,
44666929Smisaki 				    rx_msg_p->max_usage_cnt));
44673859Sml29623 			} else {
44683859Sml29623 				/* Buffer can be re-posted */
44693859Sml29623 				rx_msg_p->free = B_TRUE;
44703859Sml29623 				rx_msg_p->cur_usage_cnt = 0;
44713859Sml29623 				rx_msg_p->max_usage_cnt = 0xbaddcafe;
44723859Sml29623 				rx_msg_p->pkt_buf_size = 0;
44733859Sml29623 			}
44743859Sml29623 		}
44753859Sml29623 	}
44763859Sml29623 
44773859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel re-start..."));
44783859Sml29623 
44793859Sml29623 	status = nxge_rxdma_start_channel(nxgep, channel, rbrp, rcrp, mboxp);
44803859Sml29623 	if (status != NXGE_OK) {
44813859Sml29623 		goto fail;
44823859Sml29623 	}
44833859Sml29623 
44843859Sml29623 	MUTEX_EXIT(&rbrp->post_lock);
44853859Sml29623 	MUTEX_EXIT(&rbrp->lock);
44863859Sml29623 	MUTEX_EXIT(&rcrp->lock);
44873859Sml29623 
44883859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
44896929Smisaki 	    "Recovery Successful, RxDMAChannel#%d Restored",
44906929Smisaki 	    channel));
44913859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fatal_err_recover"));
44923859Sml29623 
44933859Sml29623 	return (NXGE_OK);
44943859Sml29623 fail:
44953859Sml29623 	MUTEX_EXIT(&rbrp->post_lock);
44963859Sml29623 	MUTEX_EXIT(&rbrp->lock);
44973859Sml29623 	MUTEX_EXIT(&rcrp->lock);
44983859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
44993859Sml29623 
45003859Sml29623 	return (NXGE_ERROR | rs);
45013859Sml29623 }
45023859Sml29623 
45033859Sml29623 nxge_status_t
45043859Sml29623 nxge_rx_port_fatal_err_recover(p_nxge_t nxgep)
45053859Sml29623 {
45066495Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
45076495Sspeer 	nxge_status_t status = NXGE_OK;
45086495Sspeer 	int rdc;
45093859Sml29623 
45103859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_port_fatal_err_recover"));
45113859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45126929Smisaki 	    "Recovering from RxPort error..."));
45136495Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disabling RxMAC...\n"));
45146495Sspeer 
45153859Sml29623 	if (nxge_rx_mac_disable(nxgep) != NXGE_OK)
45163859Sml29623 		goto fail;
45173859Sml29623 
45183859Sml29623 	NXGE_DELAY(1000);
45193859Sml29623 
45206495Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Stopping all RxDMA channels..."));
45216495Sspeer 
45226495Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
45236495Sspeer 		if ((1 << rdc) & set->owned.map) {
45246495Sspeer 			if (nxge_rxdma_fatal_err_recover(nxgep, rdc)
45256495Sspeer 			    != NXGE_OK) {
45266495Sspeer 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45276495Sspeer 				    "Could not recover channel %d", rdc));
45286495Sspeer 			}
45293859Sml29623 		}
45303859Sml29623 	}
45313859Sml29623 
45326495Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Resetting IPP..."));
45333859Sml29623 
45343859Sml29623 	/* Reset IPP */
45353859Sml29623 	if (nxge_ipp_reset(nxgep) != NXGE_OK) {
45363859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45376929Smisaki 		    "nxge_rx_port_fatal_err_recover: "
45386929Smisaki 		    "Failed to reset IPP"));
45393859Sml29623 		goto fail;
45403859Sml29623 	}
45413859Sml29623 
45423859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Reset RxMAC..."));
45433859Sml29623 
45443859Sml29623 	/* Reset RxMAC */
45453859Sml29623 	if (nxge_rx_mac_reset(nxgep) != NXGE_OK) {
45463859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45476929Smisaki 		    "nxge_rx_port_fatal_err_recover: "
45486929Smisaki 		    "Failed to reset RxMAC"));
45493859Sml29623 		goto fail;
45503859Sml29623 	}
45513859Sml29623 
45523859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize IPP..."));
45533859Sml29623 
45543859Sml29623 	/* Re-Initialize IPP */
45553859Sml29623 	if (nxge_ipp_init(nxgep) != NXGE_OK) {
45563859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45576929Smisaki 		    "nxge_rx_port_fatal_err_recover: "
45586929Smisaki 		    "Failed to init IPP"));
45593859Sml29623 		goto fail;
45603859Sml29623 	}
45613859Sml29623 
45623859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize RxMAC..."));
45633859Sml29623 
45643859Sml29623 	/* Re-Initialize RxMAC */
45653859Sml29623 	if ((status = nxge_rx_mac_init(nxgep)) != NXGE_OK) {
45663859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45676929Smisaki 		    "nxge_rx_port_fatal_err_recover: "
45686929Smisaki 		    "Failed to reset RxMAC"));
45693859Sml29623 		goto fail;
45703859Sml29623 	}
45713859Sml29623 
45723859Sml29623 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-enable RxMAC..."));
45733859Sml29623 
45743859Sml29623 	/* Re-enable RxMAC */
45753859Sml29623 	if ((status = nxge_rx_mac_enable(nxgep)) != NXGE_OK) {
45763859Sml29623 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45776929Smisaki 		    "nxge_rx_port_fatal_err_recover: "
45786929Smisaki 		    "Failed to enable RxMAC"));
45793859Sml29623 		goto fail;
45803859Sml29623 	}
45813859Sml29623 
45823859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
45836929Smisaki 	    "Recovery Successful, RxPort Restored"));
45843859Sml29623 
45853859Sml29623 	return (NXGE_OK);
45863859Sml29623 fail:
45873859Sml29623 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
45883859Sml29623 	return (status);
45893859Sml29623 }
45903859Sml29623 
45913859Sml29623 void
45923859Sml29623 nxge_rxdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan)
45933859Sml29623 {
45943859Sml29623 	rx_dma_ctl_stat_t	cs;
45953859Sml29623 	rx_ctl_dat_fifo_stat_t	cdfs;
45963859Sml29623 
45973859Sml29623 	switch (err_id) {
45983859Sml29623 	case NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR:
45993859Sml29623 	case NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR:
46003859Sml29623 	case NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR:
46013859Sml29623 	case NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR:
46023859Sml29623 	case NXGE_FM_EREPORT_RDMC_RBR_TMOUT:
46033859Sml29623 	case NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR:
46043859Sml29623 	case NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS:
46053859Sml29623 	case NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR:
46063859Sml29623 	case NXGE_FM_EREPORT_RDMC_RCRINCON:
46073859Sml29623 	case NXGE_FM_EREPORT_RDMC_RCRFULL:
46083859Sml29623 	case NXGE_FM_EREPORT_RDMC_RBRFULL:
46093859Sml29623 	case NXGE_FM_EREPORT_RDMC_RBRLOGPAGE:
46103859Sml29623 	case NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE:
46113859Sml29623 	case NXGE_FM_EREPORT_RDMC_CONFIG_ERR:
46123859Sml29623 		RXDMA_REG_READ64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG,
46136929Smisaki 		    chan, &cs.value);
46143859Sml29623 		if (err_id == NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR)
46153859Sml29623 			cs.bits.hdw.rcr_ack_err = 1;
46163859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR)
46173859Sml29623 			cs.bits.hdw.dc_fifo_err = 1;
46183859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR)
46193859Sml29623 			cs.bits.hdw.rcr_sha_par = 1;
46203859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR)
46213859Sml29623 			cs.bits.hdw.rbr_pre_par = 1;
46223859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_TMOUT)
46233859Sml29623 			cs.bits.hdw.rbr_tmout = 1;
46243859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR)
46253859Sml29623 			cs.bits.hdw.rsp_cnt_err = 1;
46263859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS)
46273859Sml29623 			cs.bits.hdw.byte_en_bus = 1;
46283859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR)
46293859Sml29623 			cs.bits.hdw.rsp_dat_err = 1;
46303859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_CONFIG_ERR)
46313859Sml29623 			cs.bits.hdw.config_err = 1;
46323859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCRINCON)
46333859Sml29623 			cs.bits.hdw.rcrincon = 1;
46343859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCRFULL)
46353859Sml29623 			cs.bits.hdw.rcrfull = 1;
46363859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBRFULL)
46373859Sml29623 			cs.bits.hdw.rbrfull = 1;
46383859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBRLOGPAGE)
46393859Sml29623 			cs.bits.hdw.rbrlogpage = 1;
46403859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE)
46413859Sml29623 			cs.bits.hdw.cfiglogpage = 1;
46425125Sjoycey #if defined(__i386)
46435125Sjoycey 		cmn_err(CE_NOTE, "!Write 0x%llx to RX_DMA_CTL_STAT_DBG_REG\n",
46446929Smisaki 		    cs.value);
46455125Sjoycey #else
46463859Sml29623 		cmn_err(CE_NOTE, "!Write 0x%lx to RX_DMA_CTL_STAT_DBG_REG\n",
46476929Smisaki 		    cs.value);
46485125Sjoycey #endif
46493859Sml29623 		RXDMA_REG_WRITE64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG,
46506929Smisaki 		    chan, cs.value);
46513859Sml29623 		break;
46523859Sml29623 	case NXGE_FM_EREPORT_RDMC_ID_MISMATCH:
46533859Sml29623 	case NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR:
46543859Sml29623 	case NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR:
46553859Sml29623 		cdfs.value = 0;
46563859Sml29623 		if (err_id ==  NXGE_FM_EREPORT_RDMC_ID_MISMATCH)
46573859Sml29623 			cdfs.bits.ldw.id_mismatch = (1 << nxgep->mac.portnum);
46583859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR)
46593859Sml29623 			cdfs.bits.ldw.zcp_eop_err = (1 << nxgep->mac.portnum);
46603859Sml29623 		else if (err_id == NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR)
46613859Sml29623 			cdfs.bits.ldw.ipp_eop_err = (1 << nxgep->mac.portnum);
46625125Sjoycey #if defined(__i386)
46635125Sjoycey 		cmn_err(CE_NOTE,
46646929Smisaki 		    "!Write 0x%llx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n",
46656929Smisaki 		    cdfs.value);
46665125Sjoycey #else
46673859Sml29623 		cmn_err(CE_NOTE,
46686929Smisaki 		    "!Write 0x%lx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n",
46696929Smisaki 		    cdfs.value);
46705125Sjoycey #endif
46716495Sspeer 		NXGE_REG_WR64(nxgep->npi_handle,
46726495Sspeer 		    RX_CTL_DAT_FIFO_STAT_DBG_REG, cdfs.value);
46733859Sml29623 		break;
46743859Sml29623 	case NXGE_FM_EREPORT_RDMC_DCF_ERR:
46753859Sml29623 		break;
46765165Syc148097 	case NXGE_FM_EREPORT_RDMC_RCR_ERR:
46773859Sml29623 		break;
46783859Sml29623 	}
46793859Sml29623 }
46806495Sspeer 
46816495Sspeer static void
46826495Sspeer nxge_rxdma_databuf_free(p_rx_rbr_ring_t rbr_p)
46836495Sspeer {
46846495Sspeer 	rxring_info_t 		*ring_info;
46856495Sspeer 	int			index;
46866495Sspeer 	uint32_t		chunk_size;
46876495Sspeer 	uint64_t		kaddr;
46886495Sspeer 	uint_t			num_blocks;
46896495Sspeer 
46906495Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_rxdma_databuf_free"));
46916495Sspeer 
46926495Sspeer 	if (rbr_p == NULL) {
46936495Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
46946495Sspeer 		    "==> nxge_rxdma_databuf_free: NULL rbr pointer"));
46956495Sspeer 		return;
46966495Sspeer 	}
46976495Sspeer 
46986495Sspeer 	if (rbr_p->rbr_alloc_type == DDI_MEM_ALLOC) {
46996495Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
47006495Sspeer 		    "==> nxge_rxdma_databuf_free: DDI"));
47016495Sspeer 		return;
47026495Sspeer 	}
47036495Sspeer 
47046495Sspeer 	ring_info = rbr_p->ring_info;
47056495Sspeer 	if (ring_info == NULL) {
47066495Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
47076495Sspeer 		    "==> nxge_rxdma_databuf_free: NULL ring info"));
47086495Sspeer 		return;
47096495Sspeer 	}
47106495Sspeer 	num_blocks = rbr_p->num_blocks;
47116495Sspeer 	for (index = 0; index < num_blocks; index++) {
47126495Sspeer 		kaddr = ring_info->buffer[index].kaddr;
47136495Sspeer 		chunk_size = ring_info->buffer[index].buf_size;
47146495Sspeer 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
47156495Sspeer 		    "==> nxge_rxdma_databuf_free: free chunk %d "
47166495Sspeer 		    "kaddrp $%p chunk size %d",
47176495Sspeer 		    index, kaddr, chunk_size));
47186495Sspeer 		if (kaddr == NULL) continue;
47196495Sspeer 		nxge_free_buf(rbr_p->rbr_alloc_type, kaddr, chunk_size);
47206495Sspeer 		ring_info->buffer[index].kaddr = NULL;
47216495Sspeer 	}
47226495Sspeer 
47236495Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_rxdma_databuf_free"));
47246495Sspeer }
47256495Sspeer 
47266495Sspeer #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
47276495Sspeer extern void contig_mem_free(void *, size_t);
47286495Sspeer #endif
47296495Sspeer 
47306495Sspeer void
47316495Sspeer nxge_free_buf(buf_alloc_type_t alloc_type, uint64_t kaddr, uint32_t buf_size)
47326495Sspeer {
47336495Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_free_buf"));
47346495Sspeer 
47356495Sspeer 	if (kaddr == NULL || !buf_size) {
47366495Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
47376495Sspeer 		    "==> nxge_free_buf: invalid kaddr $%p size to free %d",
47386495Sspeer 		    kaddr, buf_size));
47396495Sspeer 		return;
47406495Sspeer 	}
47416495Sspeer 
47426495Sspeer 	switch (alloc_type) {
47436495Sspeer 	case KMEM_ALLOC:
47446495Sspeer 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
47456495Sspeer 		    "==> nxge_free_buf: freeing kmem $%p size %d",
47466495Sspeer 		    kaddr, buf_size));
47476495Sspeer #if defined(__i386)
47486495Sspeer 		KMEM_FREE((void *)(uint32_t)kaddr, buf_size);
47496495Sspeer #else
47506495Sspeer 		KMEM_FREE((void *)kaddr, buf_size);
47516495Sspeer #endif
47526495Sspeer 		break;
47536495Sspeer 
47546495Sspeer #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
47556495Sspeer 	case CONTIG_MEM_ALLOC:
47566495Sspeer 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
47576495Sspeer 		    "==> nxge_free_buf: freeing contig_mem kaddr $%p size %d",
47586495Sspeer 		    kaddr, buf_size));
47596495Sspeer 		contig_mem_free((void *)kaddr, buf_size);
47606495Sspeer 		break;
47616495Sspeer #endif
47626495Sspeer 
47636495Sspeer 	default:
47646495Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
47656495Sspeer 		    "<== nxge_free_buf: unsupported alloc type %d",
47666495Sspeer 		    alloc_type));
47676495Sspeer 		return;
47686495Sspeer 	}
47696495Sspeer 
47706495Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_free_buf"));
47716495Sspeer }
4772