13859Sml29623 /* 23859Sml29623 * CDDL HEADER START 33859Sml29623 * 43859Sml29623 * The contents of this file are subject to the terms of the 53859Sml29623 * Common Development and Distribution License (the "License"). 63859Sml29623 * You may not use this file except in compliance with the License. 73859Sml29623 * 83859Sml29623 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 93859Sml29623 * or http://www.opensolaris.org/os/licensing. 103859Sml29623 * See the License for the specific language governing permissions 113859Sml29623 * and limitations under the License. 123859Sml29623 * 133859Sml29623 * When distributing Covered Code, include this CDDL HEADER in each 143859Sml29623 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 153859Sml29623 * If applicable, add the following below this CDDL HEADER, with the 163859Sml29623 * fields enclosed by brackets "[]" replaced with your own identifying 173859Sml29623 * information: Portions Copyright [yyyy] [name of copyright owner] 183859Sml29623 * 193859Sml29623 * CDDL HEADER END 203859Sml29623 */ 213859Sml29623 /* 225759Smisaki * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 233859Sml29623 * Use is subject to license terms. 243859Sml29623 */ 253859Sml29623 263859Sml29623 #include <sys/nxge/nxge_impl.h> 273859Sml29623 #include <sys/nxge/nxge_rxdma.h> 286495Sspeer #include <sys/nxge/nxge_hio.h> 296495Sspeer 306495Sspeer #if !defined(_BIG_ENDIAN) 316495Sspeer #include <npi_rx_rd32.h> 326495Sspeer #endif 336495Sspeer #include <npi_rx_rd64.h> 346495Sspeer #include <npi_rx_wr64.h> 353859Sml29623 363859Sml29623 #define NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp) \ 376495Sspeer (rdcgrp + nxgep->pt_config.hw_config.def_mac_rxdma_grpid) 383859Sml29623 #define NXGE_ACTUAL_RDC(nxgep, rdc) \ 393859Sml29623 (rdc + nxgep->pt_config.hw_config.start_rdc) 403859Sml29623 413859Sml29623 /* 423859Sml29623 * Globals: tunable parameters (/etc/system or adb) 433859Sml29623 * 443859Sml29623 */ 453859Sml29623 extern uint32_t nxge_rbr_size; 463859Sml29623 extern uint32_t nxge_rcr_size; 473859Sml29623 extern uint32_t nxge_rbr_spare_size; 483859Sml29623 493859Sml29623 extern uint32_t nxge_mblks_pending; 503859Sml29623 513859Sml29623 /* 523859Sml29623 * Tunable to reduce the amount of time spent in the 533859Sml29623 * ISR doing Rx Processing. 543859Sml29623 */ 553859Sml29623 extern uint32_t nxge_max_rx_pkts; 563859Sml29623 boolean_t nxge_jumbo_enable; 573859Sml29623 583859Sml29623 /* 593859Sml29623 * Tunables to manage the receive buffer blocks. 603859Sml29623 * 613859Sml29623 * nxge_rx_threshold_hi: copy all buffers. 623859Sml29623 * nxge_rx_bcopy_size_type: receive buffer block size type. 633859Sml29623 * nxge_rx_threshold_lo: copy only up to tunable block size type. 643859Sml29623 */ 653859Sml29623 extern nxge_rxbuf_threshold_t nxge_rx_threshold_hi; 663859Sml29623 extern nxge_rxbuf_type_t nxge_rx_buf_size_type; 673859Sml29623 extern nxge_rxbuf_threshold_t nxge_rx_threshold_lo; 683859Sml29623 696611Sml29623 extern uint32_t nxge_cksum_offload; 706495Sspeer 716495Sspeer static nxge_status_t nxge_map_rxdma(p_nxge_t, int); 726495Sspeer static void nxge_unmap_rxdma(p_nxge_t, int); 733859Sml29623 743859Sml29623 static nxge_status_t nxge_rxdma_hw_start_common(p_nxge_t); 756495Sspeer 766495Sspeer static nxge_status_t nxge_rxdma_hw_start(p_nxge_t, int); 776495Sspeer static void nxge_rxdma_hw_stop(p_nxge_t, int); 783859Sml29623 793859Sml29623 static nxge_status_t nxge_map_rxdma_channel(p_nxge_t, uint16_t, 803859Sml29623 p_nxge_dma_common_t *, p_rx_rbr_ring_t *, 813859Sml29623 uint32_t, 823859Sml29623 p_nxge_dma_common_t *, p_rx_rcr_ring_t *, 833859Sml29623 p_rx_mbox_t *); 843859Sml29623 static void nxge_unmap_rxdma_channel(p_nxge_t, uint16_t, 853859Sml29623 p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t); 863859Sml29623 873859Sml29623 static nxge_status_t nxge_map_rxdma_channel_cfg_ring(p_nxge_t, 883859Sml29623 uint16_t, 893859Sml29623 p_nxge_dma_common_t *, p_rx_rbr_ring_t *, 903859Sml29623 p_rx_rcr_ring_t *, p_rx_mbox_t *); 913859Sml29623 static void nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t, 923859Sml29623 p_rx_rcr_ring_t, p_rx_mbox_t); 933859Sml29623 943859Sml29623 static nxge_status_t nxge_map_rxdma_channel_buf_ring(p_nxge_t, 953859Sml29623 uint16_t, 963859Sml29623 p_nxge_dma_common_t *, 973859Sml29623 p_rx_rbr_ring_t *, uint32_t); 983859Sml29623 static void nxge_unmap_rxdma_channel_buf_ring(p_nxge_t, 993859Sml29623 p_rx_rbr_ring_t); 1003859Sml29623 1013859Sml29623 static nxge_status_t nxge_rxdma_start_channel(p_nxge_t, uint16_t, 1023859Sml29623 p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t); 1033859Sml29623 static nxge_status_t nxge_rxdma_stop_channel(p_nxge_t, uint16_t); 1043859Sml29623 1056495Sspeer static mblk_t * 1066495Sspeer nxge_rx_pkts(p_nxge_t, p_rx_rcr_ring_t, rx_dma_ctl_stat_t, int); 1073859Sml29623 1083859Sml29623 static void nxge_receive_packet(p_nxge_t, 1093859Sml29623 p_rx_rcr_ring_t, 1103859Sml29623 p_rcr_entry_t, 1113859Sml29623 boolean_t *, 1123859Sml29623 mblk_t **, mblk_t **); 1133859Sml29623 1143859Sml29623 nxge_status_t nxge_disable_rxdma_channel(p_nxge_t, uint16_t); 1153859Sml29623 1163859Sml29623 static p_rx_msg_t nxge_allocb(size_t, uint32_t, p_nxge_dma_common_t); 1173859Sml29623 static void nxge_freeb(p_rx_msg_t); 1186495Sspeer static void nxge_rx_pkts_vring(p_nxge_t, uint_t, rx_dma_ctl_stat_t); 1196495Sspeer static nxge_status_t nxge_rx_err_evnts(p_nxge_t, int, rx_dma_ctl_stat_t); 1203859Sml29623 1213859Sml29623 static nxge_status_t nxge_rxdma_handle_port_errors(p_nxge_t, 1223859Sml29623 uint32_t, uint32_t); 1233859Sml29623 1243859Sml29623 static nxge_status_t nxge_rxbuf_index_info_init(p_nxge_t, 1253859Sml29623 p_rx_rbr_ring_t); 1263859Sml29623 1273859Sml29623 1283859Sml29623 static nxge_status_t 1293859Sml29623 nxge_rxdma_fatal_err_recover(p_nxge_t, uint16_t); 1303859Sml29623 1313859Sml29623 nxge_status_t 1323859Sml29623 nxge_rx_port_fatal_err_recover(p_nxge_t); 1333859Sml29623 1346495Sspeer static void nxge_rxdma_databuf_free(p_rx_rbr_ring_t); 1356495Sspeer 1363859Sml29623 nxge_status_t 1373859Sml29623 nxge_init_rxdma_channels(p_nxge_t nxgep) 1383859Sml29623 { 1396495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 1406495Sspeer int i, count; 1413859Sml29623 1423859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_rxdma_channels")); 1433859Sml29623 1446495Sspeer if (!isLDOMguest(nxgep)) { 1456495Sspeer if (nxge_rxdma_hw_start_common(nxgep) != NXGE_OK) { 1466495Sspeer cmn_err(CE_NOTE, "hw_start_common"); 1476495Sspeer return (NXGE_ERROR); 1486495Sspeer } 1496495Sspeer } 1506495Sspeer 1516495Sspeer /* 1526495Sspeer * NXGE_LOGICAL_GROUP_MAX > NXGE_MAX_RDC_GROUPS (8) 1536495Sspeer * We only have 8 hardware RDC tables, but we may have 1546495Sspeer * up to 16 logical (software-defined) groups of RDCS, 1556495Sspeer * if we make use of layer 3 & 4 hardware classification. 1566495Sspeer */ 1576495Sspeer for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) { 1586495Sspeer if ((1 << i) & set->lg.map) { 1596495Sspeer int channel; 1606495Sspeer nxge_grp_t *group = set->group[i]; 1616495Sspeer for (channel = 0; channel < NXGE_MAX_RDCS; channel++) { 1626495Sspeer if ((1 << channel) & group->map) { 1636495Sspeer if ((nxge_grp_dc_add(nxgep, 1647755SMisaki.Kataoka@Sun.COM group, VP_BOUND_RX, channel))) 1656495Sspeer return (NXGE_ERROR); 1666495Sspeer } 1676495Sspeer } 1686495Sspeer } 1696495Sspeer if (++count == set->lg.count) 1706495Sspeer break; 1716495Sspeer } 1726495Sspeer 1736495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_rxdma_channels")); 1746495Sspeer 1756495Sspeer return (NXGE_OK); 1766495Sspeer } 1776495Sspeer 1786495Sspeer nxge_status_t 1796495Sspeer nxge_init_rxdma_channel(p_nxge_t nxge, int channel) 1806495Sspeer { 1816495Sspeer nxge_status_t status; 1826495Sspeer 1836495Sspeer NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_rxdma_channel")); 1846495Sspeer 1856495Sspeer status = nxge_map_rxdma(nxge, channel); 1863859Sml29623 if (status != NXGE_OK) { 1876495Sspeer NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL, 1886495Sspeer "<== nxge_init_rxdma: status 0x%x", status)); 1893859Sml29623 return (status); 1903859Sml29623 } 1913859Sml29623 1926495Sspeer status = nxge_rxdma_hw_start(nxge, channel); 1933859Sml29623 if (status != NXGE_OK) { 1946495Sspeer nxge_unmap_rxdma(nxge, channel); 1953859Sml29623 } 1963859Sml29623 1976495Sspeer if (!nxge->statsp->rdc_ksp[channel]) 1986495Sspeer nxge_setup_rdc_kstats(nxge, channel); 1996495Sspeer 2006495Sspeer NXGE_DEBUG_MSG((nxge, MEM2_CTL, 2016495Sspeer "<== nxge_init_rxdma_channel: status 0x%x", status)); 2023859Sml29623 2033859Sml29623 return (status); 2043859Sml29623 } 2053859Sml29623 2063859Sml29623 void 2073859Sml29623 nxge_uninit_rxdma_channels(p_nxge_t nxgep) 2083859Sml29623 { 2096495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 2106495Sspeer int rdc; 2116495Sspeer 2123859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channels")); 2133859Sml29623 2146495Sspeer if (set->owned.map == 0) { 2156495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 2166495Sspeer "nxge_uninit_rxdma_channels: no channels")); 2176495Sspeer return; 2186495Sspeer } 2196495Sspeer 2206495Sspeer for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 2216495Sspeer if ((1 << rdc) & set->owned.map) { 2226495Sspeer nxge_grp_dc_remove(nxgep, VP_BOUND_RX, rdc); 2236495Sspeer } 2246495Sspeer } 2256495Sspeer 2266495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_rxdma_channels")); 2276495Sspeer } 2286495Sspeer 2296495Sspeer void 2306495Sspeer nxge_uninit_rxdma_channel(p_nxge_t nxgep, int channel) 2316495Sspeer { 2326495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channel")); 2336495Sspeer 2346495Sspeer if (nxgep->statsp->rdc_ksp[channel]) { 2356495Sspeer kstat_delete(nxgep->statsp->rdc_ksp[channel]); 2366495Sspeer nxgep->statsp->rdc_ksp[channel] = 0; 2376495Sspeer } 2386495Sspeer 2396495Sspeer nxge_rxdma_hw_stop(nxgep, channel); 2406495Sspeer nxge_unmap_rxdma(nxgep, channel); 2416495Sspeer 2426495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uinit_rxdma_channel")); 2433859Sml29623 } 2443859Sml29623 2453859Sml29623 nxge_status_t 2463859Sml29623 nxge_reset_rxdma_channel(p_nxge_t nxgep, uint16_t channel) 2473859Sml29623 { 2483859Sml29623 npi_handle_t handle; 2493859Sml29623 npi_status_t rs = NPI_SUCCESS; 2503859Sml29623 nxge_status_t status = NXGE_OK; 2513859Sml29623 252*7812SMichael.Speer@Sun.COM NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_reset_rxdma_channel")); 2533859Sml29623 2543859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 2553859Sml29623 rs = npi_rxdma_cfg_rdc_reset(handle, channel); 2563859Sml29623 2573859Sml29623 if (rs != NPI_SUCCESS) { 2583859Sml29623 status = NXGE_ERROR | rs; 2593859Sml29623 } 2603859Sml29623 261*7812SMichael.Speer@Sun.COM NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_reset_rxdma_channel")); 262*7812SMichael.Speer@Sun.COM 2633859Sml29623 return (status); 2643859Sml29623 } 2653859Sml29623 2663859Sml29623 void 2673859Sml29623 nxge_rxdma_regs_dump_channels(p_nxge_t nxgep) 2683859Sml29623 { 2696495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 2706495Sspeer int rdc; 2713859Sml29623 2723859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_regs_dump_channels")); 2733859Sml29623 2746495Sspeer if (!isLDOMguest(nxgep)) { 2756495Sspeer npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep); 2766495Sspeer (void) npi_rxdma_dump_fzc_regs(handle); 2776495Sspeer } 2786495Sspeer 2796495Sspeer if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 2806495Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 2816495Sspeer "nxge_rxdma_regs_dump_channels: " 2826495Sspeer "NULL ring pointer(s)")); 2833859Sml29623 return; 2843859Sml29623 } 2856495Sspeer 2866495Sspeer if (set->owned.map == 0) { 2873859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 2886495Sspeer "nxge_rxdma_regs_dump_channels: no channels")); 2893859Sml29623 return; 2903859Sml29623 } 2913859Sml29623 2926495Sspeer for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 2936495Sspeer if ((1 << rdc) & set->owned.map) { 2946495Sspeer rx_rbr_ring_t *ring = 2956495Sspeer nxgep->rx_rbr_rings->rbr_rings[rdc]; 2966495Sspeer if (ring) { 2976495Sspeer (void) nxge_dump_rxdma_channel(nxgep, rdc); 2986495Sspeer } 2993859Sml29623 } 3003859Sml29623 } 3013859Sml29623 3023859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_regs_dump")); 3033859Sml29623 } 3043859Sml29623 3053859Sml29623 nxge_status_t 3063859Sml29623 nxge_dump_rxdma_channel(p_nxge_t nxgep, uint8_t channel) 3073859Sml29623 { 3083859Sml29623 npi_handle_t handle; 3093859Sml29623 npi_status_t rs = NPI_SUCCESS; 3103859Sml29623 nxge_status_t status = NXGE_OK; 3113859Sml29623 3123859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_dump_rxdma_channel")); 3133859Sml29623 3143859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 3153859Sml29623 rs = npi_rxdma_dump_rdc_regs(handle, channel); 3163859Sml29623 3173859Sml29623 if (rs != NPI_SUCCESS) { 3183859Sml29623 status = NXGE_ERROR | rs; 3193859Sml29623 } 3203859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_dump_rxdma_channel")); 3213859Sml29623 return (status); 3223859Sml29623 } 3233859Sml29623 3243859Sml29623 nxge_status_t 3253859Sml29623 nxge_init_rxdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel, 3263859Sml29623 p_rx_dma_ent_msk_t mask_p) 3273859Sml29623 { 3283859Sml29623 npi_handle_t handle; 3293859Sml29623 npi_status_t rs = NPI_SUCCESS; 3303859Sml29623 nxge_status_t status = NXGE_OK; 3313859Sml29623 3323859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 3336929Smisaki "<== nxge_init_rxdma_channel_event_mask")); 3343859Sml29623 3353859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 3363859Sml29623 rs = npi_rxdma_event_mask(handle, OP_SET, channel, mask_p); 3373859Sml29623 if (rs != NPI_SUCCESS) { 3383859Sml29623 status = NXGE_ERROR | rs; 3393859Sml29623 } 3403859Sml29623 3413859Sml29623 return (status); 3423859Sml29623 } 3433859Sml29623 3443859Sml29623 nxge_status_t 3453859Sml29623 nxge_init_rxdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel, 3463859Sml29623 p_rx_dma_ctl_stat_t cs_p) 3473859Sml29623 { 3483859Sml29623 npi_handle_t handle; 3493859Sml29623 npi_status_t rs = NPI_SUCCESS; 3503859Sml29623 nxge_status_t status = NXGE_OK; 3513859Sml29623 3523859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 3536929Smisaki "<== nxge_init_rxdma_channel_cntl_stat")); 3543859Sml29623 3553859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 3563859Sml29623 rs = npi_rxdma_control_status(handle, OP_SET, channel, cs_p); 3573859Sml29623 3583859Sml29623 if (rs != NPI_SUCCESS) { 3593859Sml29623 status = NXGE_ERROR | rs; 3603859Sml29623 } 3613859Sml29623 3623859Sml29623 return (status); 3633859Sml29623 } 3643859Sml29623 3656495Sspeer /* 3666495Sspeer * nxge_rxdma_cfg_rdcgrp_default_rdc 3676495Sspeer * 3686495Sspeer * Set the default RDC for an RDC Group (Table) 3696495Sspeer * 3706495Sspeer * Arguments: 3716495Sspeer * nxgep 3726495Sspeer * rdcgrp The group to modify 3736495Sspeer * rdc The new default RDC. 3746495Sspeer * 3756495Sspeer * Notes: 3766495Sspeer * 3776495Sspeer * NPI/NXGE function calls: 3786495Sspeer * npi_rxdma_cfg_rdc_table_default_rdc() 3796495Sspeer * 3806495Sspeer * Registers accessed: 3816495Sspeer * RDC_TBL_REG: FZC_ZCP + 0x10000 3826495Sspeer * 3836495Sspeer * Context: 3846495Sspeer * Service domain 3856495Sspeer */ 3863859Sml29623 nxge_status_t 3876495Sspeer nxge_rxdma_cfg_rdcgrp_default_rdc( 3886495Sspeer p_nxge_t nxgep, 3896495Sspeer uint8_t rdcgrp, 3906495Sspeer uint8_t rdc) 3913859Sml29623 { 3923859Sml29623 npi_handle_t handle; 3933859Sml29623 npi_status_t rs = NPI_SUCCESS; 3943859Sml29623 p_nxge_dma_pt_cfg_t p_dma_cfgp; 3953859Sml29623 p_nxge_rdc_grp_t rdc_grp_p; 3963859Sml29623 uint8_t actual_rdcgrp, actual_rdc; 3973859Sml29623 3983859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 3996929Smisaki " ==> nxge_rxdma_cfg_rdcgrp_default_rdc")); 4003859Sml29623 p_dma_cfgp = (p_nxge_dma_pt_cfg_t)&nxgep->pt_config; 4013859Sml29623 4023859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 4033859Sml29623 4046495Sspeer /* 4056495Sspeer * This has to be rewritten. Do we even allow this anymore? 4066495Sspeer */ 4073859Sml29623 rdc_grp_p = &p_dma_cfgp->rdc_grps[rdcgrp]; 4086495Sspeer RDC_MAP_IN(rdc_grp_p->map, rdc); 4096495Sspeer rdc_grp_p->def_rdc = rdc; 4103859Sml29623 4113859Sml29623 actual_rdcgrp = NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp); 4123859Sml29623 actual_rdc = NXGE_ACTUAL_RDC(nxgep, rdc); 4133859Sml29623 4146495Sspeer rs = npi_rxdma_cfg_rdc_table_default_rdc( 4156929Smisaki handle, actual_rdcgrp, actual_rdc); 4163859Sml29623 4173859Sml29623 if (rs != NPI_SUCCESS) { 4183859Sml29623 return (NXGE_ERROR | rs); 4193859Sml29623 } 4203859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 4216929Smisaki " <== nxge_rxdma_cfg_rdcgrp_default_rdc")); 4223859Sml29623 return (NXGE_OK); 4233859Sml29623 } 4243859Sml29623 4253859Sml29623 nxge_status_t 4263859Sml29623 nxge_rxdma_cfg_port_default_rdc(p_nxge_t nxgep, uint8_t port, uint8_t rdc) 4273859Sml29623 { 4283859Sml29623 npi_handle_t handle; 4293859Sml29623 4303859Sml29623 uint8_t actual_rdc; 4313859Sml29623 npi_status_t rs = NPI_SUCCESS; 4323859Sml29623 4333859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 4346929Smisaki " ==> nxge_rxdma_cfg_port_default_rdc")); 4353859Sml29623 4363859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 4376495Sspeer actual_rdc = rdc; /* XXX Hack! */ 4383859Sml29623 rs = npi_rxdma_cfg_default_port_rdc(handle, port, actual_rdc); 4393859Sml29623 4403859Sml29623 4413859Sml29623 if (rs != NPI_SUCCESS) { 4423859Sml29623 return (NXGE_ERROR | rs); 4433859Sml29623 } 4443859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 4456929Smisaki " <== nxge_rxdma_cfg_port_default_rdc")); 4463859Sml29623 4473859Sml29623 return (NXGE_OK); 4483859Sml29623 } 4493859Sml29623 4503859Sml29623 nxge_status_t 4513859Sml29623 nxge_rxdma_cfg_rcr_threshold(p_nxge_t nxgep, uint8_t channel, 4523859Sml29623 uint16_t pkts) 4533859Sml29623 { 4543859Sml29623 npi_status_t rs = NPI_SUCCESS; 4553859Sml29623 npi_handle_t handle; 4563859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 4576929Smisaki " ==> nxge_rxdma_cfg_rcr_threshold")); 4583859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 4593859Sml29623 4603859Sml29623 rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel, pkts); 4613859Sml29623 4623859Sml29623 if (rs != NPI_SUCCESS) { 4633859Sml29623 return (NXGE_ERROR | rs); 4643859Sml29623 } 4653859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_threshold")); 4663859Sml29623 return (NXGE_OK); 4673859Sml29623 } 4683859Sml29623 4693859Sml29623 nxge_status_t 4703859Sml29623 nxge_rxdma_cfg_rcr_timeout(p_nxge_t nxgep, uint8_t channel, 4713859Sml29623 uint16_t tout, uint8_t enable) 4723859Sml29623 { 4733859Sml29623 npi_status_t rs = NPI_SUCCESS; 4743859Sml29623 npi_handle_t handle; 4753859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, " ==> nxge_rxdma_cfg_rcr_timeout")); 4763859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 4773859Sml29623 if (enable == 0) { 4783859Sml29623 rs = npi_rxdma_cfg_rdc_rcr_timeout_disable(handle, channel); 4793859Sml29623 } else { 4803859Sml29623 rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 4816929Smisaki tout); 4823859Sml29623 } 4833859Sml29623 4843859Sml29623 if (rs != NPI_SUCCESS) { 4853859Sml29623 return (NXGE_ERROR | rs); 4863859Sml29623 } 4873859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_timeout")); 4883859Sml29623 return (NXGE_OK); 4893859Sml29623 } 4903859Sml29623 4913859Sml29623 nxge_status_t 4923859Sml29623 nxge_enable_rxdma_channel(p_nxge_t nxgep, uint16_t channel, 4933859Sml29623 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p) 4943859Sml29623 { 4953859Sml29623 npi_handle_t handle; 4963859Sml29623 rdc_desc_cfg_t rdc_desc; 4973859Sml29623 p_rcrcfig_b_t cfgb_p; 4983859Sml29623 npi_status_t rs = NPI_SUCCESS; 4993859Sml29623 5003859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel")); 5013859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 5023859Sml29623 /* 5033859Sml29623 * Use configuration data composed at init time. 5043859Sml29623 * Write to hardware the receive ring configurations. 5053859Sml29623 */ 5063859Sml29623 rdc_desc.mbox_enable = 1; 5073859Sml29623 rdc_desc.mbox_addr = mbox_p->mbox_addr; 5083859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 5096929Smisaki "==> nxge_enable_rxdma_channel: mboxp $%p($%p)", 5106929Smisaki mbox_p->mbox_addr, rdc_desc.mbox_addr)); 5113859Sml29623 5123859Sml29623 rdc_desc.rbr_len = rbr_p->rbb_max; 5133859Sml29623 rdc_desc.rbr_addr = rbr_p->rbr_addr; 5143859Sml29623 5153859Sml29623 switch (nxgep->rx_bksize_code) { 5163859Sml29623 case RBR_BKSIZE_4K: 5173859Sml29623 rdc_desc.page_size = SIZE_4KB; 5183859Sml29623 break; 5193859Sml29623 case RBR_BKSIZE_8K: 5203859Sml29623 rdc_desc.page_size = SIZE_8KB; 5213859Sml29623 break; 5223859Sml29623 case RBR_BKSIZE_16K: 5233859Sml29623 rdc_desc.page_size = SIZE_16KB; 5243859Sml29623 break; 5253859Sml29623 case RBR_BKSIZE_32K: 5263859Sml29623 rdc_desc.page_size = SIZE_32KB; 5273859Sml29623 break; 5283859Sml29623 } 5293859Sml29623 5303859Sml29623 rdc_desc.size0 = rbr_p->npi_pkt_buf_size0; 5313859Sml29623 rdc_desc.valid0 = 1; 5323859Sml29623 5333859Sml29623 rdc_desc.size1 = rbr_p->npi_pkt_buf_size1; 5343859Sml29623 rdc_desc.valid1 = 1; 5353859Sml29623 5363859Sml29623 rdc_desc.size2 = rbr_p->npi_pkt_buf_size2; 5373859Sml29623 rdc_desc.valid2 = 1; 5383859Sml29623 5393859Sml29623 rdc_desc.full_hdr = rcr_p->full_hdr_flag; 5403859Sml29623 rdc_desc.offset = rcr_p->sw_priv_hdr_len; 5413859Sml29623 5423859Sml29623 rdc_desc.rcr_len = rcr_p->comp_size; 5433859Sml29623 rdc_desc.rcr_addr = rcr_p->rcr_addr; 5443859Sml29623 5453859Sml29623 cfgb_p = &(rcr_p->rcr_cfgb); 5463859Sml29623 rdc_desc.rcr_threshold = cfgb_p->bits.ldw.pthres; 5476495Sspeer /* For now, disable this timeout in a guest domain. */ 5486495Sspeer if (isLDOMguest(nxgep)) { 5496495Sspeer rdc_desc.rcr_timeout = 0; 5506495Sspeer rdc_desc.rcr_timeout_enable = 0; 5516495Sspeer } else { 5526495Sspeer rdc_desc.rcr_timeout = cfgb_p->bits.ldw.timeout; 5536495Sspeer rdc_desc.rcr_timeout_enable = cfgb_p->bits.ldw.entout; 5546495Sspeer } 5553859Sml29623 5563859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: " 5576929Smisaki "rbr_len qlen %d pagesize code %d rcr_len %d", 5586929Smisaki rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len)); 5593859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: " 5606929Smisaki "size 0 %d size 1 %d size 2 %d", 5616929Smisaki rbr_p->npi_pkt_buf_size0, rbr_p->npi_pkt_buf_size1, 5626929Smisaki rbr_p->npi_pkt_buf_size2)); 5633859Sml29623 5643859Sml29623 rs = npi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc); 5653859Sml29623 if (rs != NPI_SUCCESS) { 5663859Sml29623 return (NXGE_ERROR | rs); 5673859Sml29623 } 5683859Sml29623 5693859Sml29623 /* 5703859Sml29623 * Enable the timeout and threshold. 5713859Sml29623 */ 5723859Sml29623 rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel, 5736929Smisaki rdc_desc.rcr_threshold); 5743859Sml29623 if (rs != NPI_SUCCESS) { 5753859Sml29623 return (NXGE_ERROR | rs); 5763859Sml29623 } 5773859Sml29623 5783859Sml29623 rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel, 5796929Smisaki rdc_desc.rcr_timeout); 5803859Sml29623 if (rs != NPI_SUCCESS) { 5813859Sml29623 return (NXGE_ERROR | rs); 5823859Sml29623 } 5833859Sml29623 5843859Sml29623 /* Enable the DMA */ 5853859Sml29623 rs = npi_rxdma_cfg_rdc_enable(handle, channel); 5863859Sml29623 if (rs != NPI_SUCCESS) { 5873859Sml29623 return (NXGE_ERROR | rs); 5883859Sml29623 } 5893859Sml29623 5903859Sml29623 /* Kick the DMA engine. */ 5913859Sml29623 npi_rxdma_rdc_rbr_kick(handle, channel, rbr_p->rbb_max); 5923859Sml29623 /* Clear the rbr empty bit */ 5933859Sml29623 (void) npi_rxdma_channel_rbr_empty_clear(handle, channel); 5943859Sml29623 5953859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_enable_rxdma_channel")); 5963859Sml29623 5973859Sml29623 return (NXGE_OK); 5983859Sml29623 } 5993859Sml29623 6003859Sml29623 nxge_status_t 6013859Sml29623 nxge_disable_rxdma_channel(p_nxge_t nxgep, uint16_t channel) 6023859Sml29623 { 6033859Sml29623 npi_handle_t handle; 6043859Sml29623 npi_status_t rs = NPI_SUCCESS; 6053859Sml29623 6063859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_disable_rxdma_channel")); 6073859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 6083859Sml29623 6093859Sml29623 /* disable the DMA */ 6103859Sml29623 rs = npi_rxdma_cfg_rdc_disable(handle, channel); 6113859Sml29623 if (rs != NPI_SUCCESS) { 6123859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 6136929Smisaki "<== nxge_disable_rxdma_channel:failed (0x%x)", 6146929Smisaki rs)); 6153859Sml29623 return (NXGE_ERROR | rs); 6163859Sml29623 } 6173859Sml29623 6183859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_disable_rxdma_channel")); 6193859Sml29623 return (NXGE_OK); 6203859Sml29623 } 6213859Sml29623 6223859Sml29623 nxge_status_t 6233859Sml29623 nxge_rxdma_channel_rcrflush(p_nxge_t nxgep, uint8_t channel) 6243859Sml29623 { 6253859Sml29623 npi_handle_t handle; 6263859Sml29623 nxge_status_t status = NXGE_OK; 6273859Sml29623 6283859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 6296929Smisaki "<== nxge_init_rxdma_channel_rcrflush")); 6303859Sml29623 6313859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 6323859Sml29623 npi_rxdma_rdc_rcr_flush(handle, channel); 6333859Sml29623 6343859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 6356929Smisaki "<== nxge_init_rxdma_channel_rcrflsh")); 6363859Sml29623 return (status); 6373859Sml29623 6383859Sml29623 } 6393859Sml29623 6403859Sml29623 #define MID_INDEX(l, r) ((r + l + 1) >> 1) 6413859Sml29623 6423859Sml29623 #define TO_LEFT -1 6433859Sml29623 #define TO_RIGHT 1 6443859Sml29623 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT) 6453859Sml29623 #define BOTH_LEFT (TO_LEFT + TO_LEFT) 6463859Sml29623 #define IN_MIDDLE (TO_RIGHT + TO_LEFT) 6473859Sml29623 #define NO_HINT 0xffffffff 6483859Sml29623 6493859Sml29623 /*ARGSUSED*/ 6503859Sml29623 nxge_status_t 6513859Sml29623 nxge_rxbuf_pp_to_vp(p_nxge_t nxgep, p_rx_rbr_ring_t rbr_p, 6523859Sml29623 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp, 6533859Sml29623 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index) 6543859Sml29623 { 6553859Sml29623 int bufsize; 6563859Sml29623 uint64_t pktbuf_pp; 6573859Sml29623 uint64_t dvma_addr; 6583859Sml29623 rxring_info_t *ring_info; 6593859Sml29623 int base_side, end_side; 6603859Sml29623 int r_index, l_index, anchor_index; 6613859Sml29623 int found, search_done; 6623859Sml29623 uint32_t offset, chunk_size, block_size, page_size_mask; 6633859Sml29623 uint32_t chunk_index, block_index, total_index; 6643859Sml29623 int max_iterations, iteration; 6653859Sml29623 rxbuf_index_info_t *bufinfo; 6663859Sml29623 6673859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_rxbuf_pp_to_vp")); 6683859Sml29623 6693859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 6706929Smisaki "==> nxge_rxbuf_pp_to_vp: buf_pp $%p btype %d", 6716929Smisaki pkt_buf_addr_pp, 6726929Smisaki pktbufsz_type)); 6735125Sjoycey #if defined(__i386) 6745125Sjoycey pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp; 6755125Sjoycey #else 6763859Sml29623 pktbuf_pp = (uint64_t)pkt_buf_addr_pp; 6775125Sjoycey #endif 6783859Sml29623 6793859Sml29623 switch (pktbufsz_type) { 6803859Sml29623 case 0: 6813859Sml29623 bufsize = rbr_p->pkt_buf_size0; 6823859Sml29623 break; 6833859Sml29623 case 1: 6843859Sml29623 bufsize = rbr_p->pkt_buf_size1; 6853859Sml29623 break; 6863859Sml29623 case 2: 6873859Sml29623 bufsize = rbr_p->pkt_buf_size2; 6883859Sml29623 break; 6893859Sml29623 case RCR_SINGLE_BLOCK: 6903859Sml29623 bufsize = 0; 6913859Sml29623 anchor_index = 0; 6923859Sml29623 break; 6933859Sml29623 default: 6943859Sml29623 return (NXGE_ERROR); 6953859Sml29623 } 6963859Sml29623 6973859Sml29623 if (rbr_p->num_blocks == 1) { 6983859Sml29623 anchor_index = 0; 6993859Sml29623 ring_info = rbr_p->ring_info; 7003859Sml29623 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 7013859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 7026929Smisaki "==> nxge_rxbuf_pp_to_vp: (found, 1 block) " 7036929Smisaki "buf_pp $%p btype %d anchor_index %d " 7046929Smisaki "bufinfo $%p", 7056929Smisaki pkt_buf_addr_pp, 7066929Smisaki pktbufsz_type, 7076929Smisaki anchor_index, 7086929Smisaki bufinfo)); 7093859Sml29623 7103859Sml29623 goto found_index; 7113859Sml29623 } 7123859Sml29623 7133859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 7146929Smisaki "==> nxge_rxbuf_pp_to_vp: " 7156929Smisaki "buf_pp $%p btype %d anchor_index %d", 7166929Smisaki pkt_buf_addr_pp, 7176929Smisaki pktbufsz_type, 7186929Smisaki anchor_index)); 7193859Sml29623 7203859Sml29623 ring_info = rbr_p->ring_info; 7213859Sml29623 found = B_FALSE; 7223859Sml29623 bufinfo = (rxbuf_index_info_t *)ring_info->buffer; 7233859Sml29623 iteration = 0; 7243859Sml29623 max_iterations = ring_info->max_iterations; 7253859Sml29623 /* 7263859Sml29623 * First check if this block has been seen 7273859Sml29623 * recently. This is indicated by a hint which 7283859Sml29623 * is initialized when the first buffer of the block 7293859Sml29623 * is seen. The hint is reset when the last buffer of 7303859Sml29623 * the block has been processed. 7313859Sml29623 * As three block sizes are supported, three hints 7323859Sml29623 * are kept. The idea behind the hints is that once 7333859Sml29623 * the hardware uses a block for a buffer of that 7343859Sml29623 * size, it will use it exclusively for that size 7353859Sml29623 * and will use it until it is exhausted. It is assumed 7363859Sml29623 * that there would a single block being used for the same 7373859Sml29623 * buffer sizes at any given time. 7383859Sml29623 */ 7393859Sml29623 if (ring_info->hint[pktbufsz_type] != NO_HINT) { 7403859Sml29623 anchor_index = ring_info->hint[pktbufsz_type]; 7413859Sml29623 dvma_addr = bufinfo[anchor_index].dvma_addr; 7423859Sml29623 chunk_size = bufinfo[anchor_index].buf_size; 7433859Sml29623 if ((pktbuf_pp >= dvma_addr) && 7446929Smisaki (pktbuf_pp < (dvma_addr + chunk_size))) { 7453859Sml29623 found = B_TRUE; 7463859Sml29623 /* 7473859Sml29623 * check if this is the last buffer in the block 7483859Sml29623 * If so, then reset the hint for the size; 7493859Sml29623 */ 7503859Sml29623 7513859Sml29623 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size)) 7523859Sml29623 ring_info->hint[pktbufsz_type] = NO_HINT; 7533859Sml29623 } 7543859Sml29623 } 7553859Sml29623 7563859Sml29623 if (found == B_FALSE) { 7573859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 7586929Smisaki "==> nxge_rxbuf_pp_to_vp: (!found)" 7596929Smisaki "buf_pp $%p btype %d anchor_index %d", 7606929Smisaki pkt_buf_addr_pp, 7616929Smisaki pktbufsz_type, 7626929Smisaki anchor_index)); 7633859Sml29623 7643859Sml29623 /* 7653859Sml29623 * This is the first buffer of the block of this 7663859Sml29623 * size. Need to search the whole information 7673859Sml29623 * array. 7683859Sml29623 * the search algorithm uses a binary tree search 7693859Sml29623 * algorithm. It assumes that the information is 7703859Sml29623 * already sorted with increasing order 7713859Sml29623 * info[0] < info[1] < info[2] .... < info[n-1] 7723859Sml29623 * where n is the size of the information array 7733859Sml29623 */ 7743859Sml29623 r_index = rbr_p->num_blocks - 1; 7753859Sml29623 l_index = 0; 7763859Sml29623 search_done = B_FALSE; 7773859Sml29623 anchor_index = MID_INDEX(r_index, l_index); 7783859Sml29623 while (search_done == B_FALSE) { 7793859Sml29623 if ((r_index == l_index) || 7806929Smisaki (iteration >= max_iterations)) 7813859Sml29623 search_done = B_TRUE; 7823859Sml29623 end_side = TO_RIGHT; /* to the right */ 7833859Sml29623 base_side = TO_LEFT; /* to the left */ 7843859Sml29623 /* read the DVMA address information and sort it */ 7853859Sml29623 dvma_addr = bufinfo[anchor_index].dvma_addr; 7863859Sml29623 chunk_size = bufinfo[anchor_index].buf_size; 7873859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 7886929Smisaki "==> nxge_rxbuf_pp_to_vp: (searching)" 7896929Smisaki "buf_pp $%p btype %d " 7906929Smisaki "anchor_index %d chunk_size %d dvmaaddr $%p", 7916929Smisaki pkt_buf_addr_pp, 7926929Smisaki pktbufsz_type, 7936929Smisaki anchor_index, 7946929Smisaki chunk_size, 7956929Smisaki dvma_addr)); 7963859Sml29623 7973859Sml29623 if (pktbuf_pp >= dvma_addr) 7983859Sml29623 base_side = TO_RIGHT; /* to the right */ 7993859Sml29623 if (pktbuf_pp < (dvma_addr + chunk_size)) 8003859Sml29623 end_side = TO_LEFT; /* to the left */ 8013859Sml29623 8023859Sml29623 switch (base_side + end_side) { 8036929Smisaki case IN_MIDDLE: 8046929Smisaki /* found */ 8056929Smisaki found = B_TRUE; 8066929Smisaki search_done = B_TRUE; 8076929Smisaki if ((pktbuf_pp + bufsize) < 8086929Smisaki (dvma_addr + chunk_size)) 8096929Smisaki ring_info->hint[pktbufsz_type] = 8106929Smisaki bufinfo[anchor_index].buf_index; 8116929Smisaki break; 8126929Smisaki case BOTH_RIGHT: 8136929Smisaki /* not found: go to the right */ 8146929Smisaki l_index = anchor_index + 1; 8156929Smisaki anchor_index = MID_INDEX(r_index, l_index); 8166929Smisaki break; 8176929Smisaki 8186929Smisaki case BOTH_LEFT: 8196929Smisaki /* not found: go to the left */ 8206929Smisaki r_index = anchor_index - 1; 8216929Smisaki anchor_index = MID_INDEX(r_index, l_index); 8226929Smisaki break; 8236929Smisaki default: /* should not come here */ 8246929Smisaki return (NXGE_ERROR); 8253859Sml29623 } 8263859Sml29623 iteration++; 8273859Sml29623 } 8283859Sml29623 8293859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 8306929Smisaki "==> nxge_rxbuf_pp_to_vp: (search done)" 8316929Smisaki "buf_pp $%p btype %d anchor_index %d", 8326929Smisaki pkt_buf_addr_pp, 8336929Smisaki pktbufsz_type, 8346929Smisaki anchor_index)); 8353859Sml29623 } 8363859Sml29623 8373859Sml29623 if (found == B_FALSE) { 8383859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 8396929Smisaki "==> nxge_rxbuf_pp_to_vp: (search failed)" 8406929Smisaki "buf_pp $%p btype %d anchor_index %d", 8416929Smisaki pkt_buf_addr_pp, 8426929Smisaki pktbufsz_type, 8436929Smisaki anchor_index)); 8443859Sml29623 return (NXGE_ERROR); 8453859Sml29623 } 8463859Sml29623 8473859Sml29623 found_index: 8483859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 8496929Smisaki "==> nxge_rxbuf_pp_to_vp: (FOUND1)" 8506929Smisaki "buf_pp $%p btype %d bufsize %d anchor_index %d", 8516929Smisaki pkt_buf_addr_pp, 8526929Smisaki pktbufsz_type, 8536929Smisaki bufsize, 8546929Smisaki anchor_index)); 8553859Sml29623 8563859Sml29623 /* index of the first block in this chunk */ 8573859Sml29623 chunk_index = bufinfo[anchor_index].start_index; 8583859Sml29623 dvma_addr = bufinfo[anchor_index].dvma_addr; 8593859Sml29623 page_size_mask = ring_info->block_size_mask; 8603859Sml29623 8613859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 8626929Smisaki "==> nxge_rxbuf_pp_to_vp: (FOUND3), get chunk)" 8636929Smisaki "buf_pp $%p btype %d bufsize %d " 8646929Smisaki "anchor_index %d chunk_index %d dvma $%p", 8656929Smisaki pkt_buf_addr_pp, 8666929Smisaki pktbufsz_type, 8676929Smisaki bufsize, 8686929Smisaki anchor_index, 8696929Smisaki chunk_index, 8706929Smisaki dvma_addr)); 8713859Sml29623 8723859Sml29623 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */ 8733859Sml29623 block_size = rbr_p->block_size; /* System block(page) size */ 8743859Sml29623 8753859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 8766929Smisaki "==> nxge_rxbuf_pp_to_vp: (FOUND4), get chunk)" 8776929Smisaki "buf_pp $%p btype %d bufsize %d " 8786929Smisaki "anchor_index %d chunk_index %d dvma $%p " 8796929Smisaki "offset %d block_size %d", 8806929Smisaki pkt_buf_addr_pp, 8816929Smisaki pktbufsz_type, 8826929Smisaki bufsize, 8836929Smisaki anchor_index, 8846929Smisaki chunk_index, 8856929Smisaki dvma_addr, 8866929Smisaki offset, 8876929Smisaki block_size)); 8883859Sml29623 8893859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> getting total index")); 8903859Sml29623 8913859Sml29623 block_index = (offset / block_size); /* index within chunk */ 8923859Sml29623 total_index = chunk_index + block_index; 8933859Sml29623 8943859Sml29623 8953859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 8966929Smisaki "==> nxge_rxbuf_pp_to_vp: " 8976929Smisaki "total_index %d dvma_addr $%p " 8986929Smisaki "offset %d block_size %d " 8996929Smisaki "block_index %d ", 9006929Smisaki total_index, dvma_addr, 9016929Smisaki offset, block_size, 9026929Smisaki block_index)); 9035125Sjoycey #if defined(__i386) 9045125Sjoycey *pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr + 9056929Smisaki (uint32_t)offset); 9065125Sjoycey #else 9075125Sjoycey *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr + 9086929Smisaki (uint64_t)offset); 9095125Sjoycey #endif 9103859Sml29623 9113859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 9126929Smisaki "==> nxge_rxbuf_pp_to_vp: " 9136929Smisaki "total_index %d dvma_addr $%p " 9146929Smisaki "offset %d block_size %d " 9156929Smisaki "block_index %d " 9166929Smisaki "*pkt_buf_addr_p $%p", 9176929Smisaki total_index, dvma_addr, 9186929Smisaki offset, block_size, 9196929Smisaki block_index, 9206929Smisaki *pkt_buf_addr_p)); 9213859Sml29623 9223859Sml29623 9233859Sml29623 *msg_index = total_index; 9243859Sml29623 *bufoffset = (offset & page_size_mask); 9253859Sml29623 9263859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 9276929Smisaki "==> nxge_rxbuf_pp_to_vp: get msg index: " 9286929Smisaki "msg_index %d bufoffset_index %d", 9296929Smisaki *msg_index, 9306929Smisaki *bufoffset)); 9313859Sml29623 9323859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rxbuf_pp_to_vp")); 9333859Sml29623 9343859Sml29623 return (NXGE_OK); 9353859Sml29623 } 9363859Sml29623 9373859Sml29623 /* 9383859Sml29623 * used by quick sort (qsort) function 9393859Sml29623 * to perform comparison 9403859Sml29623 */ 9413859Sml29623 static int 9423859Sml29623 nxge_sort_compare(const void *p1, const void *p2) 9433859Sml29623 { 9443859Sml29623 9453859Sml29623 rxbuf_index_info_t *a, *b; 9463859Sml29623 9473859Sml29623 a = (rxbuf_index_info_t *)p1; 9483859Sml29623 b = (rxbuf_index_info_t *)p2; 9493859Sml29623 9503859Sml29623 if (a->dvma_addr > b->dvma_addr) 9513859Sml29623 return (1); 9523859Sml29623 if (a->dvma_addr < b->dvma_addr) 9533859Sml29623 return (-1); 9543859Sml29623 return (0); 9553859Sml29623 } 9563859Sml29623 9573859Sml29623 9583859Sml29623 9593859Sml29623 /* 9603859Sml29623 * grabbed this sort implementation from common/syscall/avl.c 9613859Sml29623 * 9623859Sml29623 */ 9633859Sml29623 /* 9643859Sml29623 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified. 9653859Sml29623 * v = Ptr to array/vector of objs 9663859Sml29623 * n = # objs in the array 9673859Sml29623 * s = size of each obj (must be multiples of a word size) 9683859Sml29623 * f = ptr to function to compare two objs 9693859Sml29623 * returns (-1 = less than, 0 = equal, 1 = greater than 9703859Sml29623 */ 9713859Sml29623 void 9723859Sml29623 nxge_ksort(caddr_t v, int n, int s, int (*f)()) 9733859Sml29623 { 9743859Sml29623 int g, i, j, ii; 9753859Sml29623 unsigned int *p1, *p2; 9763859Sml29623 unsigned int tmp; 9773859Sml29623 9783859Sml29623 /* No work to do */ 9793859Sml29623 if (v == NULL || n <= 1) 9803859Sml29623 return; 9813859Sml29623 /* Sanity check on arguments */ 9823859Sml29623 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0); 9833859Sml29623 ASSERT(s > 0); 9843859Sml29623 9853859Sml29623 for (g = n / 2; g > 0; g /= 2) { 9863859Sml29623 for (i = g; i < n; i++) { 9873859Sml29623 for (j = i - g; j >= 0 && 9886929Smisaki (*f)(v + j * s, v + (j + g) * s) == 1; 9896929Smisaki j -= g) { 9903859Sml29623 p1 = (unsigned *)(v + j * s); 9913859Sml29623 p2 = (unsigned *)(v + (j + g) * s); 9923859Sml29623 for (ii = 0; ii < s / 4; ii++) { 9933859Sml29623 tmp = *p1; 9943859Sml29623 *p1++ = *p2; 9953859Sml29623 *p2++ = tmp; 9963859Sml29623 } 9973859Sml29623 } 9983859Sml29623 } 9993859Sml29623 } 10003859Sml29623 } 10013859Sml29623 10023859Sml29623 /* 10033859Sml29623 * Initialize data structures required for rxdma 10043859Sml29623 * buffer dvma->vmem address lookup 10053859Sml29623 */ 10063859Sml29623 /*ARGSUSED*/ 10073859Sml29623 static nxge_status_t 10083859Sml29623 nxge_rxbuf_index_info_init(p_nxge_t nxgep, p_rx_rbr_ring_t rbrp) 10093859Sml29623 { 10103859Sml29623 10113859Sml29623 int index; 10123859Sml29623 rxring_info_t *ring_info; 10133859Sml29623 int max_iteration = 0, max_index = 0; 10143859Sml29623 10153859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_rxbuf_index_info_init")); 10163859Sml29623 10173859Sml29623 ring_info = rbrp->ring_info; 10183859Sml29623 ring_info->hint[0] = NO_HINT; 10193859Sml29623 ring_info->hint[1] = NO_HINT; 10203859Sml29623 ring_info->hint[2] = NO_HINT; 10213859Sml29623 max_index = rbrp->num_blocks; 10223859Sml29623 10233859Sml29623 /* read the DVMA address information and sort it */ 10243859Sml29623 /* do init of the information array */ 10253859Sml29623 10263859Sml29623 10273859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA2_CTL, 10286929Smisaki " nxge_rxbuf_index_info_init Sort ptrs")); 10293859Sml29623 10303859Sml29623 /* sort the array */ 10313859Sml29623 nxge_ksort((void *)ring_info->buffer, max_index, 10326929Smisaki sizeof (rxbuf_index_info_t), nxge_sort_compare); 10333859Sml29623 10343859Sml29623 10353859Sml29623 10363859Sml29623 for (index = 0; index < max_index; index++) { 10373859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA2_CTL, 10386929Smisaki " nxge_rxbuf_index_info_init: sorted chunk %d " 10396929Smisaki " ioaddr $%p kaddr $%p size %x", 10406929Smisaki index, ring_info->buffer[index].dvma_addr, 10416929Smisaki ring_info->buffer[index].kaddr, 10426929Smisaki ring_info->buffer[index].buf_size)); 10433859Sml29623 } 10443859Sml29623 10453859Sml29623 max_iteration = 0; 10463859Sml29623 while (max_index >= (1ULL << max_iteration)) 10473859Sml29623 max_iteration++; 10483859Sml29623 ring_info->max_iterations = max_iteration + 1; 10493859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA2_CTL, 10506929Smisaki " nxge_rxbuf_index_info_init Find max iter %d", 10516929Smisaki ring_info->max_iterations)); 10523859Sml29623 10533859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxbuf_index_info_init")); 10543859Sml29623 return (NXGE_OK); 10553859Sml29623 } 10563859Sml29623 10573859Sml29623 /* ARGSUSED */ 10583859Sml29623 void 10593859Sml29623 nxge_dump_rcr_entry(p_nxge_t nxgep, p_rcr_entry_t entry_p) 10603859Sml29623 { 10613859Sml29623 #ifdef NXGE_DEBUG 10623859Sml29623 10633859Sml29623 uint32_t bptr; 10643859Sml29623 uint64_t pp; 10653859Sml29623 10663859Sml29623 bptr = entry_p->bits.hdw.pkt_buf_addr; 10673859Sml29623 10683859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 10696929Smisaki "\trcr entry $%p " 10706929Smisaki "\trcr entry 0x%0llx " 10716929Smisaki "\trcr entry 0x%08x " 10726929Smisaki "\trcr entry 0x%08x " 10736929Smisaki "\tvalue 0x%0llx\n" 10746929Smisaki "\tmulti = %d\n" 10756929Smisaki "\tpkt_type = 0x%x\n" 10766929Smisaki "\tzero_copy = %d\n" 10776929Smisaki "\tnoport = %d\n" 10786929Smisaki "\tpromis = %d\n" 10796929Smisaki "\terror = 0x%04x\n" 10806929Smisaki "\tdcf_err = 0x%01x\n" 10816929Smisaki "\tl2_len = %d\n" 10826929Smisaki "\tpktbufsize = %d\n" 10836929Smisaki "\tpkt_buf_addr = $%p\n" 10846929Smisaki "\tpkt_buf_addr (<< 6) = $%p\n", 10856929Smisaki entry_p, 10866929Smisaki *(int64_t *)entry_p, 10876929Smisaki *(int32_t *)entry_p, 10886929Smisaki *(int32_t *)((char *)entry_p + 32), 10896929Smisaki entry_p->value, 10906929Smisaki entry_p->bits.hdw.multi, 10916929Smisaki entry_p->bits.hdw.pkt_type, 10926929Smisaki entry_p->bits.hdw.zero_copy, 10936929Smisaki entry_p->bits.hdw.noport, 10946929Smisaki entry_p->bits.hdw.promis, 10956929Smisaki entry_p->bits.hdw.error, 10966929Smisaki entry_p->bits.hdw.dcf_err, 10976929Smisaki entry_p->bits.hdw.l2_len, 10986929Smisaki entry_p->bits.hdw.pktbufsz, 10996929Smisaki bptr, 11006929Smisaki entry_p->bits.ldw.pkt_buf_addr)); 11013859Sml29623 11023859Sml29623 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) << 11036929Smisaki RCR_PKT_BUF_ADDR_SHIFT; 11043859Sml29623 11053859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "rcr pp 0x%llx l2 len %d", 11066929Smisaki pp, (*(int64_t *)entry_p >> 40) & 0x3fff)); 11073859Sml29623 #endif 11083859Sml29623 } 11093859Sml29623 11103859Sml29623 void 11113859Sml29623 nxge_rxdma_regs_dump(p_nxge_t nxgep, int rdc) 11123859Sml29623 { 11133859Sml29623 npi_handle_t handle; 11143859Sml29623 rbr_stat_t rbr_stat; 11153859Sml29623 addr44_t hd_addr; 11163859Sml29623 addr44_t tail_addr; 11173859Sml29623 uint16_t qlen; 11183859Sml29623 11193859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 11206929Smisaki "==> nxge_rxdma_regs_dump: rdc channel %d", rdc)); 11213859Sml29623 11223859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 11233859Sml29623 11243859Sml29623 /* RBR head */ 11253859Sml29623 hd_addr.addr = 0; 11263859Sml29623 (void) npi_rxdma_rdc_rbr_head_get(handle, rdc, &hd_addr); 11275165Syc148097 #if defined(__i386) 11283859Sml29623 printf("nxge_rxdma_regs_dump: got hdptr $%p \n", 11296929Smisaki (void *)(uint32_t)hd_addr.addr); 11305125Sjoycey #else 11315165Syc148097 printf("nxge_rxdma_regs_dump: got hdptr $%p \n", 11326929Smisaki (void *)hd_addr.addr); 11335125Sjoycey #endif 11343859Sml29623 11353859Sml29623 /* RBR stats */ 11363859Sml29623 (void) npi_rxdma_rdc_rbr_stat_get(handle, rdc, &rbr_stat); 11373859Sml29623 printf("nxge_rxdma_regs_dump: rbr len %d \n", rbr_stat.bits.ldw.qlen); 11383859Sml29623 11393859Sml29623 /* RCR tail */ 11403859Sml29623 tail_addr.addr = 0; 11413859Sml29623 (void) npi_rxdma_rdc_rcr_tail_get(handle, rdc, &tail_addr); 11425165Syc148097 #if defined(__i386) 11433859Sml29623 printf("nxge_rxdma_regs_dump: got tail ptr $%p \n", 11446929Smisaki (void *)(uint32_t)tail_addr.addr); 11455125Sjoycey #else 11465165Syc148097 printf("nxge_rxdma_regs_dump: got tail ptr $%p \n", 11476929Smisaki (void *)tail_addr.addr); 11485125Sjoycey #endif 11493859Sml29623 11503859Sml29623 /* RCR qlen */ 11513859Sml29623 (void) npi_rxdma_rdc_rcr_qlen_get(handle, rdc, &qlen); 11523859Sml29623 printf("nxge_rxdma_regs_dump: rcr len %x \n", qlen); 11533859Sml29623 11543859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 11556929Smisaki "<== nxge_rxdma_regs_dump: rdc rdc %d", rdc)); 11563859Sml29623 } 11573859Sml29623 11583859Sml29623 void 11593859Sml29623 nxge_rxdma_stop(p_nxge_t nxgep) 11603859Sml29623 { 11613859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop")); 11623859Sml29623 11633859Sml29623 (void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP); 11643859Sml29623 (void) nxge_rx_mac_disable(nxgep); 11653859Sml29623 (void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_STOP); 11663859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop")); 11673859Sml29623 } 11683859Sml29623 11693859Sml29623 void 11703859Sml29623 nxge_rxdma_stop_reinit(p_nxge_t nxgep) 11713859Sml29623 { 11723859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_reinit")); 11733859Sml29623 11743859Sml29623 (void) nxge_rxdma_stop(nxgep); 11753859Sml29623 (void) nxge_uninit_rxdma_channels(nxgep); 11763859Sml29623 (void) nxge_init_rxdma_channels(nxgep); 11773859Sml29623 11783859Sml29623 #ifndef AXIS_DEBUG_LB 11793859Sml29623 (void) nxge_xcvr_init(nxgep); 11803859Sml29623 (void) nxge_link_monitor(nxgep, LINK_MONITOR_START); 11813859Sml29623 #endif 11823859Sml29623 (void) nxge_rx_mac_enable(nxgep); 11833859Sml29623 11843859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_reinit")); 11853859Sml29623 } 11863859Sml29623 11873859Sml29623 nxge_status_t 11883859Sml29623 nxge_rxdma_hw_mode(p_nxge_t nxgep, boolean_t enable) 11893859Sml29623 { 11906495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 11916495Sspeer nxge_status_t status; 11926495Sspeer npi_status_t rs; 11936495Sspeer int rdc; 11943859Sml29623 11953859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 11966929Smisaki "==> nxge_rxdma_hw_mode: mode %d", enable)); 11973859Sml29623 11983859Sml29623 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 11993859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 12006495Sspeer "<== nxge_rxdma_mode: not initialized")); 12013859Sml29623 return (NXGE_ERROR); 12023859Sml29623 } 12036495Sspeer 12046495Sspeer if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 12056495Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 12066495Sspeer "<== nxge_tx_port_fatal_err_recover: " 12076495Sspeer "NULL ring pointer(s)")); 12083859Sml29623 return (NXGE_ERROR); 12093859Sml29623 } 12103859Sml29623 12116495Sspeer if (set->owned.map == 0) { 12126495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, 12136495Sspeer "nxge_rxdma_regs_dump_channels: no channels")); 12146495Sspeer return (NULL); 12156495Sspeer } 12166495Sspeer 12176495Sspeer for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 12186495Sspeer if ((1 << rdc) & set->owned.map) { 12196495Sspeer rx_rbr_ring_t *ring = 12206495Sspeer nxgep->rx_rbr_rings->rbr_rings[rdc]; 12216495Sspeer npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep); 12226495Sspeer if (ring) { 12236495Sspeer if (enable) { 12246495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 12256495Sspeer "==> nxge_rxdma_hw_mode: " 12266495Sspeer "channel %d (enable)", rdc)); 12276495Sspeer rs = npi_rxdma_cfg_rdc_enable 12286495Sspeer (handle, rdc); 12296495Sspeer } else { 12306495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 12316495Sspeer "==> nxge_rxdma_hw_mode: " 12326495Sspeer "channel %d disable)", rdc)); 12336495Sspeer rs = npi_rxdma_cfg_rdc_disable 12346495Sspeer (handle, rdc); 12356495Sspeer } 12366495Sspeer } 12373859Sml29623 } 12383859Sml29623 } 12393859Sml29623 12403859Sml29623 status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs); 12413859Sml29623 12423859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 12436929Smisaki "<== nxge_rxdma_hw_mode: status 0x%x", status)); 12443859Sml29623 12453859Sml29623 return (status); 12463859Sml29623 } 12473859Sml29623 12483859Sml29623 void 12493859Sml29623 nxge_rxdma_enable_channel(p_nxge_t nxgep, uint16_t channel) 12503859Sml29623 { 12513859Sml29623 npi_handle_t handle; 12523859Sml29623 12533859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 12546929Smisaki "==> nxge_rxdma_enable_channel: channel %d", channel)); 12553859Sml29623 12563859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 12573859Sml29623 (void) npi_rxdma_cfg_rdc_enable(handle, channel); 12583859Sml29623 12593859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_enable_channel")); 12603859Sml29623 } 12613859Sml29623 12623859Sml29623 void 12633859Sml29623 nxge_rxdma_disable_channel(p_nxge_t nxgep, uint16_t channel) 12643859Sml29623 { 12653859Sml29623 npi_handle_t handle; 12663859Sml29623 12673859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, 12686929Smisaki "==> nxge_rxdma_disable_channel: channel %d", channel)); 12693859Sml29623 12703859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 12713859Sml29623 (void) npi_rxdma_cfg_rdc_disable(handle, channel); 12723859Sml29623 12733859Sml29623 NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_disable_channel")); 12743859Sml29623 } 12753859Sml29623 12763859Sml29623 void 12773859Sml29623 nxge_hw_start_rx(p_nxge_t nxgep) 12783859Sml29623 { 12793859Sml29623 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_hw_start_rx")); 12803859Sml29623 12813859Sml29623 (void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_START); 12823859Sml29623 (void) nxge_rx_mac_enable(nxgep); 12833859Sml29623 12843859Sml29623 NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_hw_start_rx")); 12853859Sml29623 } 12863859Sml29623 12873859Sml29623 /*ARGSUSED*/ 12883859Sml29623 void 12893859Sml29623 nxge_fixup_rxdma_rings(p_nxge_t nxgep) 12903859Sml29623 { 12916495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 12926495Sspeer int rdc; 12933859Sml29623 12943859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_fixup_rxdma_rings")); 12953859Sml29623 12966495Sspeer if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 12976495Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 12986495Sspeer "<== nxge_tx_port_fatal_err_recover: " 12996495Sspeer "NULL ring pointer(s)")); 13003859Sml29623 return; 13013859Sml29623 } 13023859Sml29623 13036495Sspeer if (set->owned.map == 0) { 13043859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 13056495Sspeer "nxge_rxdma_regs_dump_channels: no channels")); 13063859Sml29623 return; 13073859Sml29623 } 13086495Sspeer 13096495Sspeer for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 13106495Sspeer if ((1 << rdc) & set->owned.map) { 13116495Sspeer rx_rbr_ring_t *ring = 13126495Sspeer nxgep->rx_rbr_rings->rbr_rings[rdc]; 13136495Sspeer if (ring) { 13146495Sspeer nxge_rxdma_hw_stop(nxgep, rdc); 13156495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, 13166929Smisaki "==> nxge_fixup_rxdma_rings: " 13176929Smisaki "channel %d ring $%px", 13186929Smisaki rdc, ring)); 13196495Sspeer (void) nxge_rxdma_fixup_channel 13206495Sspeer (nxgep, rdc, rdc); 13216495Sspeer } 13226495Sspeer } 13233859Sml29623 } 13243859Sml29623 13253859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_fixup_rxdma_rings")); 13263859Sml29623 } 13273859Sml29623 13283859Sml29623 void 13293859Sml29623 nxge_rxdma_fix_channel(p_nxge_t nxgep, uint16_t channel) 13303859Sml29623 { 13313859Sml29623 int i; 13323859Sml29623 13333859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fix_channel")); 13343859Sml29623 i = nxge_rxdma_get_ring_index(nxgep, channel); 13353859Sml29623 if (i < 0) { 13363859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 13376929Smisaki "<== nxge_rxdma_fix_channel: no entry found")); 13383859Sml29623 return; 13393859Sml29623 } 13403859Sml29623 13413859Sml29623 nxge_rxdma_fixup_channel(nxgep, channel, i); 13423859Sml29623 13436495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fix_channel")); 13443859Sml29623 } 13453859Sml29623 13463859Sml29623 void 13473859Sml29623 nxge_rxdma_fixup_channel(p_nxge_t nxgep, uint16_t channel, int entry) 13483859Sml29623 { 13493859Sml29623 int ndmas; 13503859Sml29623 p_rx_rbr_rings_t rx_rbr_rings; 13513859Sml29623 p_rx_rbr_ring_t *rbr_rings; 13523859Sml29623 p_rx_rcr_rings_t rx_rcr_rings; 13533859Sml29623 p_rx_rcr_ring_t *rcr_rings; 13543859Sml29623 p_rx_mbox_areas_t rx_mbox_areas_p; 13553859Sml29623 p_rx_mbox_t *rx_mbox_p; 13563859Sml29623 p_nxge_dma_pool_t dma_buf_poolp; 13573859Sml29623 p_nxge_dma_pool_t dma_cntl_poolp; 13583859Sml29623 p_rx_rbr_ring_t rbrp; 13593859Sml29623 p_rx_rcr_ring_t rcrp; 13603859Sml29623 p_rx_mbox_t mboxp; 13613859Sml29623 p_nxge_dma_common_t dmap; 13623859Sml29623 nxge_status_t status = NXGE_OK; 13633859Sml29623 13643859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fixup_channel")); 13653859Sml29623 13663859Sml29623 (void) nxge_rxdma_stop_channel(nxgep, channel); 13673859Sml29623 13683859Sml29623 dma_buf_poolp = nxgep->rx_buf_pool_p; 13693859Sml29623 dma_cntl_poolp = nxgep->rx_cntl_pool_p; 13703859Sml29623 13713859Sml29623 if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) { 13723859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 13736929Smisaki "<== nxge_rxdma_fixup_channel: buf not allocated")); 13743859Sml29623 return; 13753859Sml29623 } 13763859Sml29623 13773859Sml29623 ndmas = dma_buf_poolp->ndmas; 13783859Sml29623 if (!ndmas) { 13793859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 13806929Smisaki "<== nxge_rxdma_fixup_channel: no dma allocated")); 13813859Sml29623 return; 13823859Sml29623 } 13833859Sml29623 13843859Sml29623 rx_rbr_rings = nxgep->rx_rbr_rings; 13853859Sml29623 rx_rcr_rings = nxgep->rx_rcr_rings; 13863859Sml29623 rbr_rings = rx_rbr_rings->rbr_rings; 13873859Sml29623 rcr_rings = rx_rcr_rings->rcr_rings; 13883859Sml29623 rx_mbox_areas_p = nxgep->rx_mbox_areas_p; 13893859Sml29623 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 13903859Sml29623 13913859Sml29623 /* Reinitialize the receive block and completion rings */ 13923859Sml29623 rbrp = (p_rx_rbr_ring_t)rbr_rings[entry], 13936929Smisaki rcrp = (p_rx_rcr_ring_t)rcr_rings[entry], 13946929Smisaki mboxp = (p_rx_mbox_t)rx_mbox_p[entry]; 13953859Sml29623 13963859Sml29623 13973859Sml29623 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 13983859Sml29623 rbrp->rbr_rd_index = 0; 13993859Sml29623 rcrp->comp_rd_index = 0; 14003859Sml29623 rcrp->comp_wt_index = 0; 14013859Sml29623 14023859Sml29623 dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc; 14033859Sml29623 bzero((caddr_t)dmap->kaddrp, dmap->alength); 14043859Sml29623 14053859Sml29623 status = nxge_rxdma_start_channel(nxgep, channel, 14066929Smisaki rbrp, rcrp, mboxp); 14073859Sml29623 if (status != NXGE_OK) { 14083859Sml29623 goto nxge_rxdma_fixup_channel_fail; 14093859Sml29623 } 14103859Sml29623 if (status != NXGE_OK) { 14113859Sml29623 goto nxge_rxdma_fixup_channel_fail; 14123859Sml29623 } 14133859Sml29623 14143859Sml29623 nxge_rxdma_fixup_channel_fail: 14153859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 14166929Smisaki "==> nxge_rxdma_fixup_channel: failed (0x%08x)", status)); 14173859Sml29623 14183859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fixup_channel")); 14193859Sml29623 } 14203859Sml29623 14216495Sspeer /* ARGSUSED */ 14223859Sml29623 int 14233859Sml29623 nxge_rxdma_get_ring_index(p_nxge_t nxgep, uint16_t channel) 14243859Sml29623 { 14256495Sspeer return (channel); 14263859Sml29623 } 14273859Sml29623 14283859Sml29623 p_rx_rbr_ring_t 14293859Sml29623 nxge_rxdma_get_rbr_ring(p_nxge_t nxgep, uint16_t channel) 14303859Sml29623 { 14316495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 14326495Sspeer nxge_channel_t rdc; 14333859Sml29623 14343859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 14356929Smisaki "==> nxge_rxdma_get_rbr_ring: channel %d", channel)); 14363859Sml29623 14376495Sspeer if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) { 14386495Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 14396495Sspeer "<== nxge_rxdma_get_rbr_ring: " 14406495Sspeer "NULL ring pointer(s)")); 14413859Sml29623 return (NULL); 14423859Sml29623 } 14436495Sspeer 14446495Sspeer if (set->owned.map == 0) { 14453859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 14466495Sspeer "<== nxge_rxdma_get_rbr_ring: no channels")); 14473859Sml29623 return (NULL); 14483859Sml29623 } 14493859Sml29623 14506495Sspeer for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 14516495Sspeer if ((1 << rdc) & set->owned.map) { 14526495Sspeer rx_rbr_ring_t *ring = 14536495Sspeer nxgep->rx_rbr_rings->rbr_rings[rdc]; 14546495Sspeer if (ring) { 14556495Sspeer if (channel == ring->rdc) { 14566495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, 14576495Sspeer "==> nxge_rxdma_get_rbr_ring: " 14586495Sspeer "channel %d ring $%p", rdc, ring)); 14596495Sspeer return (ring); 14606495Sspeer } 14616495Sspeer } 14623859Sml29623 } 14633859Sml29623 } 14643859Sml29623 14653859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 14666929Smisaki "<== nxge_rxdma_get_rbr_ring: not found")); 14673859Sml29623 14683859Sml29623 return (NULL); 14693859Sml29623 } 14703859Sml29623 14713859Sml29623 p_rx_rcr_ring_t 14723859Sml29623 nxge_rxdma_get_rcr_ring(p_nxge_t nxgep, uint16_t channel) 14733859Sml29623 { 14746495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 14756495Sspeer nxge_channel_t rdc; 14763859Sml29623 14773859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 14786929Smisaki "==> nxge_rxdma_get_rcr_ring: channel %d", channel)); 14793859Sml29623 14806495Sspeer if (nxgep->rx_rcr_rings == 0 || nxgep->rx_rcr_rings->rcr_rings == 0) { 14816495Sspeer NXGE_DEBUG_MSG((nxgep, TX_CTL, 14826495Sspeer "<== nxge_rxdma_get_rcr_ring: " 14836495Sspeer "NULL ring pointer(s)")); 14843859Sml29623 return (NULL); 14853859Sml29623 } 14866495Sspeer 14876495Sspeer if (set->owned.map == 0) { 14883859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 14896495Sspeer "<== nxge_rxdma_get_rbr_ring: no channels")); 14903859Sml29623 return (NULL); 14913859Sml29623 } 14923859Sml29623 14936495Sspeer for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 14946495Sspeer if ((1 << rdc) & set->owned.map) { 14956495Sspeer rx_rcr_ring_t *ring = 14966495Sspeer nxgep->rx_rcr_rings->rcr_rings[rdc]; 14976495Sspeer if (ring) { 14986495Sspeer if (channel == ring->rdc) { 14996495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, 15006495Sspeer "==> nxge_rxdma_get_rcr_ring: " 15016495Sspeer "channel %d ring $%p", rdc, ring)); 15026495Sspeer return (ring); 15036495Sspeer } 15046495Sspeer } 15053859Sml29623 } 15063859Sml29623 } 15073859Sml29623 15083859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 15096929Smisaki "<== nxge_rxdma_get_rcr_ring: not found")); 15103859Sml29623 15113859Sml29623 return (NULL); 15123859Sml29623 } 15133859Sml29623 15143859Sml29623 /* 15153859Sml29623 * Static functions start here. 15163859Sml29623 */ 15173859Sml29623 static p_rx_msg_t 15183859Sml29623 nxge_allocb(size_t size, uint32_t pri, p_nxge_dma_common_t dmabuf_p) 15193859Sml29623 { 15203859Sml29623 p_rx_msg_t nxge_mp = NULL; 15213859Sml29623 p_nxge_dma_common_t dmamsg_p; 15223859Sml29623 uchar_t *buffer; 15233859Sml29623 15243859Sml29623 nxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP); 15253859Sml29623 if (nxge_mp == NULL) { 15264185Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 15276929Smisaki "Allocation of a rx msg failed.")); 15283859Sml29623 goto nxge_allocb_exit; 15293859Sml29623 } 15303859Sml29623 15313859Sml29623 nxge_mp->use_buf_pool = B_FALSE; 15323859Sml29623 if (dmabuf_p) { 15333859Sml29623 nxge_mp->use_buf_pool = B_TRUE; 15343859Sml29623 dmamsg_p = (p_nxge_dma_common_t)&nxge_mp->buf_dma; 15353859Sml29623 *dmamsg_p = *dmabuf_p; 15363859Sml29623 dmamsg_p->nblocks = 1; 15373859Sml29623 dmamsg_p->block_size = size; 15383859Sml29623 dmamsg_p->alength = size; 15393859Sml29623 buffer = (uchar_t *)dmabuf_p->kaddrp; 15403859Sml29623 15413859Sml29623 dmabuf_p->kaddrp = (void *) 15426929Smisaki ((char *)dmabuf_p->kaddrp + size); 15433859Sml29623 dmabuf_p->ioaddr_pp = (void *) 15446929Smisaki ((char *)dmabuf_p->ioaddr_pp + size); 15453859Sml29623 dmabuf_p->alength -= size; 15463859Sml29623 dmabuf_p->offset += size; 15473859Sml29623 dmabuf_p->dma_cookie.dmac_laddress += size; 15483859Sml29623 dmabuf_p->dma_cookie.dmac_size -= size; 15493859Sml29623 15503859Sml29623 } else { 15513859Sml29623 buffer = KMEM_ALLOC(size, KM_NOSLEEP); 15523859Sml29623 if (buffer == NULL) { 15534185Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 15546929Smisaki "Allocation of a receive page failed.")); 15553859Sml29623 goto nxge_allocb_fail1; 15563859Sml29623 } 15573859Sml29623 } 15583859Sml29623 15593859Sml29623 nxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &nxge_mp->freeb); 15603859Sml29623 if (nxge_mp->rx_mblk_p == NULL) { 15614185Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "desballoc failed.")); 15623859Sml29623 goto nxge_allocb_fail2; 15633859Sml29623 } 15643859Sml29623 15653859Sml29623 nxge_mp->buffer = buffer; 15663859Sml29623 nxge_mp->block_size = size; 15673859Sml29623 nxge_mp->freeb.free_func = (void (*)())nxge_freeb; 15683859Sml29623 nxge_mp->freeb.free_arg = (caddr_t)nxge_mp; 15693859Sml29623 nxge_mp->ref_cnt = 1; 15703859Sml29623 nxge_mp->free = B_TRUE; 15713859Sml29623 nxge_mp->rx_use_bcopy = B_FALSE; 15723859Sml29623 15733859Sml29623 atomic_inc_32(&nxge_mblks_pending); 15743859Sml29623 15753859Sml29623 goto nxge_allocb_exit; 15763859Sml29623 15773859Sml29623 nxge_allocb_fail2: 15783859Sml29623 if (!nxge_mp->use_buf_pool) { 15793859Sml29623 KMEM_FREE(buffer, size); 15803859Sml29623 } 15813859Sml29623 15823859Sml29623 nxge_allocb_fail1: 15833859Sml29623 KMEM_FREE(nxge_mp, sizeof (rx_msg_t)); 15843859Sml29623 nxge_mp = NULL; 15853859Sml29623 15863859Sml29623 nxge_allocb_exit: 15873859Sml29623 return (nxge_mp); 15883859Sml29623 } 15893859Sml29623 15903859Sml29623 p_mblk_t 15913859Sml29623 nxge_dupb(p_rx_msg_t nxge_mp, uint_t offset, size_t size) 15923859Sml29623 { 15933859Sml29623 p_mblk_t mp; 15943859Sml29623 15953859Sml29623 NXGE_DEBUG_MSG((NULL, MEM_CTL, "==> nxge_dupb")); 15963859Sml29623 NXGE_DEBUG_MSG((NULL, MEM_CTL, "nxge_mp = $%p " 15976929Smisaki "offset = 0x%08X " 15986929Smisaki "size = 0x%08X", 15996929Smisaki nxge_mp, offset, size)); 16003859Sml29623 16013859Sml29623 mp = desballoc(&nxge_mp->buffer[offset], size, 16026929Smisaki 0, &nxge_mp->freeb); 16033859Sml29623 if (mp == NULL) { 16043859Sml29623 NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 16053859Sml29623 goto nxge_dupb_exit; 16063859Sml29623 } 16073859Sml29623 atomic_inc_32(&nxge_mp->ref_cnt); 16083859Sml29623 16093859Sml29623 16103859Sml29623 nxge_dupb_exit: 16113859Sml29623 NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p", 16126929Smisaki nxge_mp)); 16133859Sml29623 return (mp); 16143859Sml29623 } 16153859Sml29623 16163859Sml29623 p_mblk_t 16173859Sml29623 nxge_dupb_bcopy(p_rx_msg_t nxge_mp, uint_t offset, size_t size) 16183859Sml29623 { 16193859Sml29623 p_mblk_t mp; 16203859Sml29623 uchar_t *dp; 16213859Sml29623 16223859Sml29623 mp = allocb(size + NXGE_RXBUF_EXTRA, 0); 16233859Sml29623 if (mp == NULL) { 16243859Sml29623 NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed")); 16253859Sml29623 goto nxge_dupb_bcopy_exit; 16263859Sml29623 } 16273859Sml29623 dp = mp->b_rptr = mp->b_rptr + NXGE_RXBUF_EXTRA; 16283859Sml29623 bcopy((void *)&nxge_mp->buffer[offset], dp, size); 16293859Sml29623 mp->b_wptr = dp + size; 16303859Sml29623 16313859Sml29623 nxge_dupb_bcopy_exit: 16323859Sml29623 NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p", 16336929Smisaki nxge_mp)); 16343859Sml29623 return (mp); 16353859Sml29623 } 16363859Sml29623 16373859Sml29623 void nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p, 16383859Sml29623 p_rx_msg_t rx_msg_p); 16393859Sml29623 16403859Sml29623 void 16413859Sml29623 nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p) 16423859Sml29623 { 16433859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_post_page")); 16443859Sml29623 16453859Sml29623 /* Reuse this buffer */ 16463859Sml29623 rx_msg_p->free = B_FALSE; 16473859Sml29623 rx_msg_p->cur_usage_cnt = 0; 16483859Sml29623 rx_msg_p->max_usage_cnt = 0; 16493859Sml29623 rx_msg_p->pkt_buf_size = 0; 16503859Sml29623 16513859Sml29623 if (rx_rbr_p->rbr_use_bcopy) { 16523859Sml29623 rx_msg_p->rx_use_bcopy = B_FALSE; 16533859Sml29623 atomic_dec_32(&rx_rbr_p->rbr_consumed); 16543859Sml29623 } 16553859Sml29623 16563859Sml29623 /* 16573859Sml29623 * Get the rbr header pointer and its offset index. 16583859Sml29623 */ 16593859Sml29623 MUTEX_ENTER(&rx_rbr_p->post_lock); 16603859Sml29623 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) & 16616929Smisaki rx_rbr_p->rbr_wrap_mask); 16623859Sml29623 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr; 16633859Sml29623 MUTEX_EXIT(&rx_rbr_p->post_lock); 16645770Sml29623 npi_rxdma_rdc_rbr_kick(NXGE_DEV_NPI_HANDLE(nxgep), 16655770Sml29623 rx_rbr_p->rdc, 1); 16663859Sml29623 16673859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 16686929Smisaki "<== nxge_post_page (channel %d post_next_index %d)", 16696929Smisaki rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index)); 16703859Sml29623 16713859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_post_page")); 16723859Sml29623 } 16733859Sml29623 16743859Sml29623 void 16753859Sml29623 nxge_freeb(p_rx_msg_t rx_msg_p) 16763859Sml29623 { 16773859Sml29623 size_t size; 16783859Sml29623 uchar_t *buffer = NULL; 16793859Sml29623 int ref_cnt; 16804874Sml29623 boolean_t free_state = B_FALSE; 16813859Sml29623 16825170Stm144005 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p; 16835170Stm144005 16843859Sml29623 NXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> nxge_freeb")); 16853859Sml29623 NXGE_DEBUG_MSG((NULL, MEM2_CTL, 16866929Smisaki "nxge_freeb:rx_msg_p = $%p (block pending %d)", 16876929Smisaki rx_msg_p, nxge_mblks_pending)); 16883859Sml29623 16894874Sml29623 /* 16904874Sml29623 * First we need to get the free state, then 16914874Sml29623 * atomic decrement the reference count to prevent 16924874Sml29623 * the race condition with the interrupt thread that 16934874Sml29623 * is processing a loaned up buffer block. 16944874Sml29623 */ 16954874Sml29623 free_state = rx_msg_p->free; 16963859Sml29623 ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1); 16973859Sml29623 if (!ref_cnt) { 16985770Sml29623 atomic_dec_32(&nxge_mblks_pending); 16993859Sml29623 buffer = rx_msg_p->buffer; 17003859Sml29623 size = rx_msg_p->block_size; 17013859Sml29623 NXGE_DEBUG_MSG((NULL, MEM2_CTL, "nxge_freeb: " 17026929Smisaki "will free: rx_msg_p = $%p (block pending %d)", 17036929Smisaki rx_msg_p, nxge_mblks_pending)); 17043859Sml29623 17053859Sml29623 if (!rx_msg_p->use_buf_pool) { 17063859Sml29623 KMEM_FREE(buffer, size); 17073859Sml29623 } 17083859Sml29623 17093859Sml29623 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t)); 17105170Stm144005 17115759Smisaki if (ring) { 17125759Smisaki /* 17135759Smisaki * Decrement the receive buffer ring's reference 17145759Smisaki * count, too. 17155759Smisaki */ 17165759Smisaki atomic_dec_32(&ring->rbr_ref_cnt); 17175759Smisaki 17185759Smisaki /* 17196495Sspeer * Free the receive buffer ring, if 17205759Smisaki * 1. all the receive buffers have been freed 17215759Smisaki * 2. and we are in the proper state (that is, 17225759Smisaki * we are not UNMAPPING). 17235759Smisaki */ 17245759Smisaki if (ring->rbr_ref_cnt == 0 && 17255759Smisaki ring->rbr_state == RBR_UNMAPPED) { 17266495Sspeer /* 17276495Sspeer * Free receive data buffers, 17286495Sspeer * buffer index information 17296495Sspeer * (rxring_info) and 17306495Sspeer * the message block ring. 17316495Sspeer */ 17326495Sspeer NXGE_DEBUG_MSG((NULL, RX_CTL, 17336495Sspeer "nxge_freeb:rx_msg_p = $%p " 17346495Sspeer "(block pending %d) free buffers", 17356495Sspeer rx_msg_p, nxge_mblks_pending)); 17366495Sspeer nxge_rxdma_databuf_free(ring); 17376495Sspeer if (ring->ring_info) { 17386495Sspeer KMEM_FREE(ring->ring_info, 17396495Sspeer sizeof (rxring_info_t)); 17406495Sspeer } 17416495Sspeer 17426495Sspeer if (ring->rx_msg_ring) { 17436495Sspeer KMEM_FREE(ring->rx_msg_ring, 17446495Sspeer ring->tnblocks * 17456495Sspeer sizeof (p_rx_msg_t)); 17466495Sspeer } 17475759Smisaki KMEM_FREE(ring, sizeof (*ring)); 17485759Smisaki } 17495170Stm144005 } 17503859Sml29623 return; 17513859Sml29623 } 17523859Sml29623 17533859Sml29623 /* 17543859Sml29623 * Repost buffer. 17553859Sml29623 */ 17565759Smisaki if (free_state && (ref_cnt == 1) && ring) { 17573859Sml29623 NXGE_DEBUG_MSG((NULL, RX_CTL, 17583859Sml29623 "nxge_freeb: post page $%p:", rx_msg_p)); 17595170Stm144005 if (ring->rbr_state == RBR_POSTING) 17605170Stm144005 nxge_post_page(rx_msg_p->nxgep, ring, rx_msg_p); 17613859Sml29623 } 17623859Sml29623 17633859Sml29623 NXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== nxge_freeb")); 17643859Sml29623 } 17653859Sml29623 17663859Sml29623 uint_t 17673859Sml29623 nxge_rx_intr(void *arg1, void *arg2) 17683859Sml29623 { 17693859Sml29623 p_nxge_ldv_t ldvp = (p_nxge_ldv_t)arg1; 17703859Sml29623 p_nxge_t nxgep = (p_nxge_t)arg2; 17713859Sml29623 p_nxge_ldg_t ldgp; 17723859Sml29623 uint8_t channel; 17733859Sml29623 npi_handle_t handle; 17743859Sml29623 rx_dma_ctl_stat_t cs; 17753859Sml29623 17763859Sml29623 #ifdef NXGE_DEBUG 17773859Sml29623 rxdma_cfig1_t cfg; 17783859Sml29623 #endif 17793859Sml29623 uint_t serviced = DDI_INTR_UNCLAIMED; 17803859Sml29623 17813859Sml29623 if (ldvp == NULL) { 17823859Sml29623 NXGE_DEBUG_MSG((NULL, INT_CTL, 17836929Smisaki "<== nxge_rx_intr: arg2 $%p arg1 $%p", 17846929Smisaki nxgep, ldvp)); 17853859Sml29623 17863859Sml29623 return (DDI_INTR_CLAIMED); 17873859Sml29623 } 17883859Sml29623 17893859Sml29623 if (arg2 == NULL || (void *)ldvp->nxgep != arg2) { 17903859Sml29623 nxgep = ldvp->nxgep; 17913859Sml29623 } 17926602Sspeer 17936602Sspeer if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) || 17946602Sspeer (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) { 17956602Sspeer NXGE_DEBUG_MSG((nxgep, INT_CTL, 17966602Sspeer "<== nxge_rx_intr: interface not started or intialized")); 17976602Sspeer return (DDI_INTR_CLAIMED); 17986602Sspeer } 17996602Sspeer 18003859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 18016929Smisaki "==> nxge_rx_intr: arg2 $%p arg1 $%p", 18026929Smisaki nxgep, ldvp)); 18033859Sml29623 18043859Sml29623 /* 18053859Sml29623 * This interrupt handler is for a specific 18063859Sml29623 * receive dma channel. 18073859Sml29623 */ 18083859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 18093859Sml29623 /* 18103859Sml29623 * Get the control and status for this channel. 18113859Sml29623 */ 18123859Sml29623 channel = ldvp->channel; 18133859Sml29623 ldgp = ldvp->ldgp; 18143859Sml29623 RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, channel, &cs.value); 18153859Sml29623 18163859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_intr:channel %d " 18176929Smisaki "cs 0x%016llx rcrto 0x%x rcrthres %x", 18186929Smisaki channel, 18196929Smisaki cs.value, 18206929Smisaki cs.bits.hdw.rcrto, 18216929Smisaki cs.bits.hdw.rcrthres)); 18223859Sml29623 18236495Sspeer nxge_rx_pkts_vring(nxgep, ldvp->vdma_index, cs); 18243859Sml29623 serviced = DDI_INTR_CLAIMED; 18253859Sml29623 18263859Sml29623 /* error events. */ 18273859Sml29623 if (cs.value & RX_DMA_CTL_STAT_ERROR) { 18286495Sspeer (void) nxge_rx_err_evnts(nxgep, channel, cs); 18293859Sml29623 } 18303859Sml29623 18313859Sml29623 nxge_intr_exit: 18323859Sml29623 /* 18333859Sml29623 * Enable the mailbox update interrupt if we want 18343859Sml29623 * to use mailbox. We probably don't need to use 18353859Sml29623 * mailbox as it only saves us one pio read. 18363859Sml29623 * Also write 1 to rcrthres and rcrto to clear 18373859Sml29623 * these two edge triggered bits. 18383859Sml29623 */ 18393859Sml29623 18403859Sml29623 cs.value &= RX_DMA_CTL_STAT_WR1C; 18413859Sml29623 cs.bits.hdw.mex = 1; 18423859Sml29623 RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel, 18436929Smisaki cs.value); 18443859Sml29623 18453859Sml29623 /* 18463859Sml29623 * Rearm this logical group if this is a single device 18473859Sml29623 * group. 18483859Sml29623 */ 18493859Sml29623 if (ldgp->nldvs == 1) { 18503859Sml29623 ldgimgm_t mgm; 18513859Sml29623 mgm.value = 0; 18523859Sml29623 mgm.bits.ldw.arm = 1; 18533859Sml29623 mgm.bits.ldw.timer = ldgp->ldg_timer; 18546495Sspeer if (isLDOMguest(nxgep)) { 18556495Sspeer nxge_hio_ldgimgn(nxgep, ldgp); 18566495Sspeer } else { 18576495Sspeer NXGE_REG_WR64(handle, 18583859Sml29623 LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg), 18593859Sml29623 mgm.value); 18606495Sspeer } 18613859Sml29623 } 18623859Sml29623 18633859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_intr: serviced %d", 18646929Smisaki serviced)); 18653859Sml29623 return (serviced); 18663859Sml29623 } 18673859Sml29623 18683859Sml29623 /* 18693859Sml29623 * Process the packets received in the specified logical device 18703859Sml29623 * and pass up a chain of message blocks to the upper layer. 18713859Sml29623 */ 18723859Sml29623 static void 18736495Sspeer nxge_rx_pkts_vring(p_nxge_t nxgep, uint_t vindex, rx_dma_ctl_stat_t cs) 18743859Sml29623 { 18753859Sml29623 p_mblk_t mp; 18763859Sml29623 p_rx_rcr_ring_t rcrp; 18773859Sml29623 18783859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts_vring")); 18796495Sspeer rcrp = nxgep->rx_rcr_rings->rcr_rings[vindex]; 18806495Sspeer if (rcrp->poll_flag) { 18816495Sspeer /* It is in the poll mode */ 18826495Sspeer return; 18836495Sspeer } 18846495Sspeer 18856495Sspeer if ((mp = nxge_rx_pkts(nxgep, rcrp, cs, -1)) == NULL) { 18863859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 18876929Smisaki "<== nxge_rx_pkts_vring: no mp")); 18883859Sml29623 return; 18893859Sml29623 } 18903859Sml29623 18913859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts_vring: $%p", 18926929Smisaki mp)); 18933859Sml29623 18943859Sml29623 #ifdef NXGE_DEBUG 18953859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 18966929Smisaki "==> nxge_rx_pkts_vring:calling mac_rx " 18976929Smisaki "LEN %d mp $%p mp->b_cont $%p mp->b_next $%p rcrp $%p " 18986929Smisaki "mac_handle $%p", 18996929Smisaki mp->b_wptr - mp->b_rptr, 19006929Smisaki mp, mp->b_cont, mp->b_next, 19016929Smisaki rcrp, rcrp->rcr_mac_handle)); 19023859Sml29623 19033859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 19046929Smisaki "==> nxge_rx_pkts_vring: dump packets " 19056929Smisaki "(mp $%p b_rptr $%p b_wptr $%p):\n %s", 19066929Smisaki mp, 19076929Smisaki mp->b_rptr, 19086929Smisaki mp->b_wptr, 19096929Smisaki nxge_dump_packet((char *)mp->b_rptr, 19106929Smisaki mp->b_wptr - mp->b_rptr))); 19113859Sml29623 if (mp->b_cont) { 19123859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 19136929Smisaki "==> nxge_rx_pkts_vring: dump b_cont packets " 19146929Smisaki "(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s", 19156929Smisaki mp->b_cont, 19166929Smisaki mp->b_cont->b_rptr, 19176929Smisaki mp->b_cont->b_wptr, 19186929Smisaki nxge_dump_packet((char *)mp->b_cont->b_rptr, 19196929Smisaki mp->b_cont->b_wptr - mp->b_cont->b_rptr))); 19203859Sml29623 } 19213859Sml29623 if (mp->b_next) { 19223859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 19236929Smisaki "==> nxge_rx_pkts_vring: dump next packets " 19246929Smisaki "(b_rptr $%p): %s", 19256929Smisaki mp->b_next->b_rptr, 19266929Smisaki nxge_dump_packet((char *)mp->b_next->b_rptr, 19276929Smisaki mp->b_next->b_wptr - mp->b_next->b_rptr))); 19283859Sml29623 } 19293859Sml29623 #endif 19303859Sml29623 19316495Sspeer if (!isLDOMguest(nxgep)) 19326495Sspeer mac_rx(nxgep->mach, rcrp->rcr_mac_handle, mp); 19336495Sspeer #if defined(sun4v) 19346495Sspeer else { /* isLDOMguest(nxgep) */ 19356495Sspeer nxge_hio_data_t *nhd = (nxge_hio_data_t *) 19366495Sspeer nxgep->nxge_hw_p->hio; 19376495Sspeer nx_vio_fp_t *vio = &nhd->hio.vio; 19386495Sspeer 19396495Sspeer if (vio->cb.vio_net_rx_cb) { 19406495Sspeer (*vio->cb.vio_net_rx_cb) 19416495Sspeer (nxgep->hio_vr->vhp, mp); 19426495Sspeer } 19436495Sspeer } 19446495Sspeer #endif 19453859Sml29623 } 19463859Sml29623 19473859Sml29623 19483859Sml29623 /* 19493859Sml29623 * This routine is the main packet receive processing function. 19503859Sml29623 * It gets the packet type, error code, and buffer related 19513859Sml29623 * information from the receive completion entry. 19523859Sml29623 * How many completion entries to process is based on the number of packets 19533859Sml29623 * queued by the hardware, a hardware maintained tail pointer 19543859Sml29623 * and a configurable receive packet count. 19553859Sml29623 * 19563859Sml29623 * A chain of message blocks will be created as result of processing 19573859Sml29623 * the completion entries. This chain of message blocks will be returned and 19583859Sml29623 * a hardware control status register will be updated with the number of 19593859Sml29623 * packets were removed from the hardware queue. 19603859Sml29623 * 19613859Sml29623 */ 19626495Sspeer static mblk_t * 19636495Sspeer nxge_rx_pkts(p_nxge_t nxgep, p_rx_rcr_ring_t rcr_p, rx_dma_ctl_stat_t cs, 19646495Sspeer int bytes_to_pickup) 19653859Sml29623 { 19663859Sml29623 npi_handle_t handle; 19673859Sml29623 uint8_t channel; 19683859Sml29623 uint32_t comp_rd_index; 19693859Sml29623 p_rcr_entry_t rcr_desc_rd_head_p; 19703859Sml29623 p_rcr_entry_t rcr_desc_rd_head_pp; 19713859Sml29623 p_mblk_t nmp, mp_cont, head_mp, *tail_mp; 19723859Sml29623 uint16_t qlen, nrcr_read, npkt_read; 19736495Sspeer uint32_t qlen_hw; 19743859Sml29623 boolean_t multi; 19756495Sspeer rcrcfig_b_t rcr_cfg_b; 19766495Sspeer int totallen = 0; 19773859Sml29623 #if defined(_BIG_ENDIAN) 19783859Sml29623 npi_status_t rs = NPI_SUCCESS; 19793859Sml29623 #endif 19803859Sml29623 19816495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: " 19826929Smisaki "channel %d", rcr_p->rdc)); 19833859Sml29623 19843859Sml29623 if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) { 19853859Sml29623 return (NULL); 19863859Sml29623 } 19873859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 19883859Sml29623 channel = rcr_p->rdc; 19893859Sml29623 19903859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 19916929Smisaki "==> nxge_rx_pkts: START: rcr channel %d " 19926929Smisaki "head_p $%p head_pp $%p index %d ", 19936929Smisaki channel, rcr_p->rcr_desc_rd_head_p, 19946929Smisaki rcr_p->rcr_desc_rd_head_pp, 19956929Smisaki rcr_p->comp_rd_index)); 19963859Sml29623 19973859Sml29623 19983859Sml29623 #if !defined(_BIG_ENDIAN) 19993859Sml29623 qlen = RXDMA_REG_READ32(handle, RCRSTAT_A_REG, channel) & 0xffff; 20003859Sml29623 #else 20013859Sml29623 rs = npi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen); 20023859Sml29623 if (rs != NPI_SUCCESS) { 20036495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: " 20043859Sml29623 "channel %d, get qlen failed 0x%08x", 20056929Smisaki channel, rs)); 20063859Sml29623 return (NULL); 20073859Sml29623 } 20083859Sml29623 #endif 20093859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts:rcr channel %d " 20106929Smisaki "qlen %d", channel, qlen)); 20113859Sml29623 20123859Sml29623 20133859Sml29623 20143859Sml29623 if (!qlen) { 20153859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 20166929Smisaki "==> nxge_rx_pkts:rcr channel %d " 20176929Smisaki "qlen %d (no pkts)", channel, qlen)); 20183859Sml29623 20193859Sml29623 return (NULL); 20203859Sml29623 } 20213859Sml29623 20223859Sml29623 comp_rd_index = rcr_p->comp_rd_index; 20233859Sml29623 20243859Sml29623 rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p; 20253859Sml29623 rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp; 20263859Sml29623 nrcr_read = npkt_read = 0; 20273859Sml29623 20283859Sml29623 /* 20293859Sml29623 * Number of packets queued 20303859Sml29623 * (The jumbo or multi packet will be counted as only one 20313859Sml29623 * packets and it may take up more than one completion entry). 20323859Sml29623 */ 20333859Sml29623 qlen_hw = (qlen < nxge_max_rx_pkts) ? 20346929Smisaki qlen : nxge_max_rx_pkts; 20353859Sml29623 head_mp = NULL; 20363859Sml29623 tail_mp = &head_mp; 20373859Sml29623 nmp = mp_cont = NULL; 20383859Sml29623 multi = B_FALSE; 20393859Sml29623 20403859Sml29623 while (qlen_hw) { 20413859Sml29623 20423859Sml29623 #ifdef NXGE_DEBUG 20433859Sml29623 nxge_dump_rcr_entry(nxgep, rcr_desc_rd_head_p); 20443859Sml29623 #endif 20453859Sml29623 /* 20463859Sml29623 * Process one completion ring entry. 20473859Sml29623 */ 20483859Sml29623 nxge_receive_packet(nxgep, 20496929Smisaki rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont); 20503859Sml29623 20513859Sml29623 /* 20523859Sml29623 * message chaining modes 20533859Sml29623 */ 20543859Sml29623 if (nmp) { 20553859Sml29623 nmp->b_next = NULL; 20563859Sml29623 if (!multi && !mp_cont) { /* frame fits a partition */ 20573859Sml29623 *tail_mp = nmp; 20583859Sml29623 tail_mp = &nmp->b_next; 20596495Sspeer totallen += MBLKL(nmp); 20603859Sml29623 nmp = NULL; 20613859Sml29623 } else if (multi && !mp_cont) { /* first segment */ 20623859Sml29623 *tail_mp = nmp; 20633859Sml29623 tail_mp = &nmp->b_cont; 20646495Sspeer totallen += MBLKL(nmp); 20653859Sml29623 } else if (multi && mp_cont) { /* mid of multi segs */ 20663859Sml29623 *tail_mp = mp_cont; 20673859Sml29623 tail_mp = &mp_cont->b_cont; 20686495Sspeer totallen += MBLKL(mp_cont); 20693859Sml29623 } else if (!multi && mp_cont) { /* last segment */ 20703859Sml29623 *tail_mp = mp_cont; 20713859Sml29623 tail_mp = &nmp->b_next; 20726495Sspeer totallen += MBLKL(mp_cont); 20733859Sml29623 nmp = NULL; 20743859Sml29623 } 20753859Sml29623 } 20763859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 20776929Smisaki "==> nxge_rx_pkts: loop: rcr channel %d " 20786929Smisaki "before updating: multi %d " 20796929Smisaki "nrcr_read %d " 20806929Smisaki "npk read %d " 20816929Smisaki "head_pp $%p index %d ", 20826929Smisaki channel, 20836929Smisaki multi, 20846929Smisaki nrcr_read, npkt_read, rcr_desc_rd_head_pp, 20856929Smisaki comp_rd_index)); 20863859Sml29623 20873859Sml29623 if (!multi) { 20883859Sml29623 qlen_hw--; 20893859Sml29623 npkt_read++; 20903859Sml29623 } 20913859Sml29623 20923859Sml29623 /* 20933859Sml29623 * Update the next read entry. 20943859Sml29623 */ 20953859Sml29623 comp_rd_index = NEXT_ENTRY(comp_rd_index, 20966929Smisaki rcr_p->comp_wrap_mask); 20973859Sml29623 20983859Sml29623 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p, 20996929Smisaki rcr_p->rcr_desc_first_p, 21006929Smisaki rcr_p->rcr_desc_last_p); 21013859Sml29623 21023859Sml29623 nrcr_read++; 21033859Sml29623 21043859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 21056929Smisaki "<== nxge_rx_pkts: (SAM, process one packet) " 21066929Smisaki "nrcr_read %d", 21076929Smisaki nrcr_read)); 21083859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 21096929Smisaki "==> nxge_rx_pkts: loop: rcr channel %d " 21106929Smisaki "multi %d " 21116929Smisaki "nrcr_read %d " 21126929Smisaki "npk read %d " 21136929Smisaki "head_pp $%p index %d ", 21146929Smisaki channel, 21156929Smisaki multi, 21166929Smisaki nrcr_read, npkt_read, rcr_desc_rd_head_pp, 21176929Smisaki comp_rd_index)); 21183859Sml29623 21196495Sspeer if ((bytes_to_pickup != -1) && 21206495Sspeer (totallen >= bytes_to_pickup)) { 21216495Sspeer break; 21226495Sspeer } 21233859Sml29623 } 21243859Sml29623 21253859Sml29623 rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp; 21263859Sml29623 rcr_p->comp_rd_index = comp_rd_index; 21273859Sml29623 rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p; 21283859Sml29623 21293859Sml29623 if ((nxgep->intr_timeout != rcr_p->intr_timeout) || 21306929Smisaki (nxgep->intr_threshold != rcr_p->intr_threshold)) { 21313859Sml29623 rcr_p->intr_timeout = nxgep->intr_timeout; 21323859Sml29623 rcr_p->intr_threshold = nxgep->intr_threshold; 21333859Sml29623 rcr_cfg_b.value = 0x0ULL; 21343859Sml29623 if (rcr_p->intr_timeout) 21353859Sml29623 rcr_cfg_b.bits.ldw.entout = 1; 21363859Sml29623 rcr_cfg_b.bits.ldw.timeout = rcr_p->intr_timeout; 21373859Sml29623 rcr_cfg_b.bits.ldw.pthres = rcr_p->intr_threshold; 21383859Sml29623 RXDMA_REG_WRITE64(handle, RCRCFIG_B_REG, 21396929Smisaki channel, rcr_cfg_b.value); 21403859Sml29623 } 21413859Sml29623 21423859Sml29623 cs.bits.ldw.pktread = npkt_read; 21433859Sml29623 cs.bits.ldw.ptrread = nrcr_read; 21443859Sml29623 RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, 21456929Smisaki channel, cs.value); 21463859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 21476929Smisaki "==> nxge_rx_pkts: EXIT: rcr channel %d " 21486929Smisaki "head_pp $%p index %016llx ", 21496929Smisaki channel, 21506929Smisaki rcr_p->rcr_desc_rd_head_pp, 21516929Smisaki rcr_p->comp_rd_index)); 21523859Sml29623 /* 21533859Sml29623 * Update RCR buffer pointer read and number of packets 21543859Sml29623 * read. 21553859Sml29623 */ 21563859Sml29623 21573859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_pkts")); 21583859Sml29623 return (head_mp); 21593859Sml29623 } 21603859Sml29623 21613859Sml29623 void 21623859Sml29623 nxge_receive_packet(p_nxge_t nxgep, 21633859Sml29623 p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p, 21643859Sml29623 boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont) 21653859Sml29623 { 21663859Sml29623 p_mblk_t nmp = NULL; 21673859Sml29623 uint64_t multi; 21683859Sml29623 uint64_t dcf_err; 21693859Sml29623 uint8_t channel; 21703859Sml29623 21713859Sml29623 boolean_t first_entry = B_TRUE; 21723859Sml29623 boolean_t is_tcp_udp = B_FALSE; 21733859Sml29623 boolean_t buffer_free = B_FALSE; 21743859Sml29623 boolean_t error_send_up = B_FALSE; 21753859Sml29623 uint8_t error_type; 21763859Sml29623 uint16_t l2_len; 21773859Sml29623 uint16_t skip_len; 21783859Sml29623 uint8_t pktbufsz_type; 21793859Sml29623 uint64_t rcr_entry; 21803859Sml29623 uint64_t *pkt_buf_addr_pp; 21813859Sml29623 uint64_t *pkt_buf_addr_p; 21823859Sml29623 uint32_t buf_offset; 21833859Sml29623 uint32_t bsize; 21843859Sml29623 uint32_t error_disp_cnt; 21853859Sml29623 uint32_t msg_index; 21863859Sml29623 p_rx_rbr_ring_t rx_rbr_p; 21873859Sml29623 p_rx_msg_t *rx_msg_ring_p; 21883859Sml29623 p_rx_msg_t rx_msg_p; 21893859Sml29623 uint16_t sw_offset_bytes = 0, hdr_size = 0; 21903859Sml29623 nxge_status_t status = NXGE_OK; 21913859Sml29623 boolean_t is_valid = B_FALSE; 21923859Sml29623 p_nxge_rx_ring_stats_t rdc_stats; 21933859Sml29623 uint32_t bytes_read; 21943859Sml29623 uint64_t pkt_type; 21953859Sml29623 uint64_t frag; 21966028Ssbehera boolean_t pkt_too_long_err = B_FALSE; 21973859Sml29623 #ifdef NXGE_DEBUG 21983859Sml29623 int dump_len; 21993859Sml29623 #endif 22003859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_receive_packet")); 22013859Sml29623 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE; 22023859Sml29623 22033859Sml29623 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p); 22043859Sml29623 22053859Sml29623 multi = (rcr_entry & RCR_MULTI_MASK); 22063859Sml29623 dcf_err = (rcr_entry & RCR_DCF_ERROR_MASK); 22073859Sml29623 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK); 22083859Sml29623 22093859Sml29623 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT); 22103859Sml29623 frag = (rcr_entry & RCR_FRAG_MASK); 22113859Sml29623 22123859Sml29623 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT); 22133859Sml29623 22143859Sml29623 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >> 22156929Smisaki RCR_PKTBUFSZ_SHIFT); 22165125Sjoycey #if defined(__i386) 22175125Sjoycey pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry & 22186929Smisaki RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT); 22195125Sjoycey #else 22203859Sml29623 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) << 22216929Smisaki RCR_PKT_BUF_ADDR_SHIFT); 22225125Sjoycey #endif 22233859Sml29623 22243859Sml29623 channel = rcr_p->rdc; 22253859Sml29623 22263859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 22276929Smisaki "==> nxge_receive_packet: entryp $%p entry 0x%0llx " 22286929Smisaki "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx " 22296929Smisaki "error_type 0x%x pkt_type 0x%x " 22306929Smisaki "pktbufsz_type %d ", 22316929Smisaki rcr_desc_rd_head_p, 22326929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len, 22336929Smisaki multi, 22346929Smisaki error_type, 22356929Smisaki pkt_type, 22366929Smisaki pktbufsz_type)); 22373859Sml29623 22383859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 22396929Smisaki "==> nxge_receive_packet: entryp $%p entry 0x%0llx " 22406929Smisaki "pkt_buf_addr_pp $%p l2_len %d multi 0x%llx " 22416929Smisaki "error_type 0x%x pkt_type 0x%x ", rcr_desc_rd_head_p, 22426929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len, 22436929Smisaki multi, 22446929Smisaki error_type, 22456929Smisaki pkt_type)); 22463859Sml29623 22473859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 22486929Smisaki "==> (rbr) nxge_receive_packet: entry 0x%0llx " 22496929Smisaki "full pkt_buf_addr_pp $%p l2_len %d", 22506929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len)); 22513859Sml29623 22523859Sml29623 /* get the stats ptr */ 22533859Sml29623 rdc_stats = rcr_p->rdc_stats; 22543859Sml29623 22553859Sml29623 if (!l2_len) { 22563859Sml29623 22573859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 22586929Smisaki "<== nxge_receive_packet: failed: l2 length is 0.")); 22593859Sml29623 return; 22603859Sml29623 } 22613859Sml29623 22626028Ssbehera /* 22636028Ssbehera * Sofware workaround for BMAC hardware limitation that allows 22646028Ssbehera * maxframe size of 1526, instead of 1522 for non-jumbo and 0x2406 22656028Ssbehera * instead of 0x2400 for jumbo. 22666028Ssbehera */ 22676028Ssbehera if (l2_len > nxgep->mac.maxframesize) { 22686028Ssbehera pkt_too_long_err = B_TRUE; 22696028Ssbehera } 22706028Ssbehera 22714185Sspeer /* Hardware sends us 4 bytes of CRC as no stripping is done. */ 22724185Sspeer l2_len -= ETHERFCSL; 22734185Sspeer 22743859Sml29623 /* shift 6 bits to get the full io address */ 22755125Sjoycey #if defined(__i386) 22765125Sjoycey pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp << 22776929Smisaki RCR_PKT_BUF_ADDR_SHIFT_FULL); 22785125Sjoycey #else 22793859Sml29623 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp << 22806929Smisaki RCR_PKT_BUF_ADDR_SHIFT_FULL); 22815125Sjoycey #endif 22823859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 22836929Smisaki "==> (rbr) nxge_receive_packet: entry 0x%0llx " 22846929Smisaki "full pkt_buf_addr_pp $%p l2_len %d", 22856929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len)); 22863859Sml29623 22873859Sml29623 rx_rbr_p = rcr_p->rx_rbr_p; 22883859Sml29623 rx_msg_ring_p = rx_rbr_p->rx_msg_ring; 22893859Sml29623 22903859Sml29623 if (first_entry) { 22913859Sml29623 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL : 22926929Smisaki RXDMA_HDR_SIZE_DEFAULT); 22933859Sml29623 22943859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 22956929Smisaki "==> nxge_receive_packet: first entry 0x%016llx " 22966929Smisaki "pkt_buf_addr_pp $%p l2_len %d hdr %d", 22976929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len, 22986929Smisaki hdr_size)); 22993859Sml29623 } 23003859Sml29623 23013859Sml29623 MUTEX_ENTER(&rcr_p->lock); 23023859Sml29623 MUTEX_ENTER(&rx_rbr_p->lock); 23033859Sml29623 23043859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 23056929Smisaki "==> (rbr 1) nxge_receive_packet: entry 0x%0llx " 23066929Smisaki "full pkt_buf_addr_pp $%p l2_len %d", 23076929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len)); 23083859Sml29623 23093859Sml29623 /* 23103859Sml29623 * Packet buffer address in the completion entry points 23113859Sml29623 * to the starting buffer address (offset 0). 23123859Sml29623 * Use the starting buffer address to locate the corresponding 23133859Sml29623 * kernel address. 23143859Sml29623 */ 23153859Sml29623 status = nxge_rxbuf_pp_to_vp(nxgep, rx_rbr_p, 23166929Smisaki pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p, 23176929Smisaki &buf_offset, 23186929Smisaki &msg_index); 23193859Sml29623 23203859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 23216929Smisaki "==> (rbr 2) nxge_receive_packet: entry 0x%0llx " 23226929Smisaki "full pkt_buf_addr_pp $%p l2_len %d", 23236929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len)); 23243859Sml29623 23253859Sml29623 if (status != NXGE_OK) { 23263859Sml29623 MUTEX_EXIT(&rx_rbr_p->lock); 23273859Sml29623 MUTEX_EXIT(&rcr_p->lock); 23283859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 23296929Smisaki "<== nxge_receive_packet: found vaddr failed %d", 23306929Smisaki status)); 23313859Sml29623 return; 23323859Sml29623 } 23333859Sml29623 23343859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 23356929Smisaki "==> (rbr 3) nxge_receive_packet: entry 0x%0llx " 23366929Smisaki "full pkt_buf_addr_pp $%p l2_len %d", 23376929Smisaki rcr_entry, pkt_buf_addr_pp, l2_len)); 23383859Sml29623 23393859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 23406929Smisaki "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx " 23416929Smisaki "full pkt_buf_addr_pp $%p l2_len %d", 23426929Smisaki msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 23433859Sml29623 23443859Sml29623 rx_msg_p = rx_msg_ring_p[msg_index]; 23453859Sml29623 23463859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 23476929Smisaki "==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx " 23486929Smisaki "full pkt_buf_addr_pp $%p l2_len %d", 23496929Smisaki msg_index, rcr_entry, pkt_buf_addr_pp, l2_len)); 23503859Sml29623 23513859Sml29623 switch (pktbufsz_type) { 23523859Sml29623 case RCR_PKTBUFSZ_0: 23533859Sml29623 bsize = rx_rbr_p->pkt_buf_size0_bytes; 23543859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 23556929Smisaki "==> nxge_receive_packet: 0 buf %d", bsize)); 23563859Sml29623 break; 23573859Sml29623 case RCR_PKTBUFSZ_1: 23583859Sml29623 bsize = rx_rbr_p->pkt_buf_size1_bytes; 23593859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 23606929Smisaki "==> nxge_receive_packet: 1 buf %d", bsize)); 23613859Sml29623 break; 23623859Sml29623 case RCR_PKTBUFSZ_2: 23633859Sml29623 bsize = rx_rbr_p->pkt_buf_size2_bytes; 23643859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 23656929Smisaki "==> nxge_receive_packet: 2 buf %d", bsize)); 23663859Sml29623 break; 23673859Sml29623 case RCR_SINGLE_BLOCK: 23683859Sml29623 bsize = rx_msg_p->block_size; 23693859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 23706929Smisaki "==> nxge_receive_packet: single %d", bsize)); 23713859Sml29623 23723859Sml29623 break; 23733859Sml29623 default: 23743859Sml29623 MUTEX_EXIT(&rx_rbr_p->lock); 23753859Sml29623 MUTEX_EXIT(&rcr_p->lock); 23763859Sml29623 return; 23773859Sml29623 } 23783859Sml29623 23793859Sml29623 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma, 23806929Smisaki (buf_offset + sw_offset_bytes), 23816929Smisaki (hdr_size + l2_len), 23826929Smisaki DDI_DMA_SYNC_FORCPU); 23833859Sml29623 23843859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 23856929Smisaki "==> nxge_receive_packet: after first dump:usage count")); 23863859Sml29623 23873859Sml29623 if (rx_msg_p->cur_usage_cnt == 0) { 23883859Sml29623 if (rx_rbr_p->rbr_use_bcopy) { 23893859Sml29623 atomic_inc_32(&rx_rbr_p->rbr_consumed); 23903859Sml29623 if (rx_rbr_p->rbr_consumed < 23916929Smisaki rx_rbr_p->rbr_threshold_hi) { 23923859Sml29623 if (rx_rbr_p->rbr_threshold_lo == 0 || 23936929Smisaki ((rx_rbr_p->rbr_consumed >= 23946929Smisaki rx_rbr_p->rbr_threshold_lo) && 23956929Smisaki (rx_rbr_p->rbr_bufsize_type >= 23966929Smisaki pktbufsz_type))) { 23973859Sml29623 rx_msg_p->rx_use_bcopy = B_TRUE; 23983859Sml29623 } 23993859Sml29623 } else { 24003859Sml29623 rx_msg_p->rx_use_bcopy = B_TRUE; 24013859Sml29623 } 24023859Sml29623 } 24033859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 24046929Smisaki "==> nxge_receive_packet: buf %d (new block) ", 24056929Smisaki bsize)); 24063859Sml29623 24073859Sml29623 rx_msg_p->pkt_buf_size_code = pktbufsz_type; 24083859Sml29623 rx_msg_p->pkt_buf_size = bsize; 24093859Sml29623 rx_msg_p->cur_usage_cnt = 1; 24103859Sml29623 if (pktbufsz_type == RCR_SINGLE_BLOCK) { 24113859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 24126929Smisaki "==> nxge_receive_packet: buf %d " 24136929Smisaki "(single block) ", 24146929Smisaki bsize)); 24153859Sml29623 /* 24163859Sml29623 * Buffer can be reused once the free function 24173859Sml29623 * is called. 24183859Sml29623 */ 24193859Sml29623 rx_msg_p->max_usage_cnt = 1; 24203859Sml29623 buffer_free = B_TRUE; 24213859Sml29623 } else { 24223859Sml29623 rx_msg_p->max_usage_cnt = rx_msg_p->block_size/bsize; 24233859Sml29623 if (rx_msg_p->max_usage_cnt == 1) { 24243859Sml29623 buffer_free = B_TRUE; 24253859Sml29623 } 24263859Sml29623 } 24273859Sml29623 } else { 24283859Sml29623 rx_msg_p->cur_usage_cnt++; 24293859Sml29623 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) { 24303859Sml29623 buffer_free = B_TRUE; 24313859Sml29623 } 24323859Sml29623 } 24333859Sml29623 24343859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 24353859Sml29623 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ", 24366929Smisaki msg_index, l2_len, 24376929Smisaki rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt)); 24383859Sml29623 24396028Ssbehera if ((error_type) || (dcf_err) || (pkt_too_long_err)) { 24403859Sml29623 rdc_stats->ierrors++; 24413859Sml29623 if (dcf_err) { 24423859Sml29623 rdc_stats->dcf_err++; 24433859Sml29623 #ifdef NXGE_DEBUG 24443859Sml29623 if (!rdc_stats->dcf_err) { 24453859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 24463859Sml29623 "nxge_receive_packet: channel %d dcf_err rcr" 24473859Sml29623 " 0x%llx", channel, rcr_entry)); 24483859Sml29623 } 24493859Sml29623 #endif 24503859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL, 24516929Smisaki NXGE_FM_EREPORT_RDMC_DCF_ERR); 24526028Ssbehera } else if (pkt_too_long_err) { 24536028Ssbehera rdc_stats->pkt_too_long_err++; 24546028Ssbehera NXGE_DEBUG_MSG((nxgep, RX_CTL, " nxge_receive_packet:" 24556028Ssbehera " channel %d packet length [%d] > " 24566028Ssbehera "maxframesize [%d]", channel, l2_len + ETHERFCSL, 24576028Ssbehera nxgep->mac.maxframesize)); 24583859Sml29623 } else { 24593859Sml29623 /* Update error stats */ 24603859Sml29623 error_disp_cnt = NXGE_ERROR_SHOW_MAX; 24613859Sml29623 rdc_stats->errlog.compl_err_type = error_type; 24623859Sml29623 24633859Sml29623 switch (error_type) { 24645523Syc148097 /* 24655523Syc148097 * Do not send FMA ereport for RCR_L2_ERROR and 24665523Syc148097 * RCR_L4_CSUM_ERROR because most likely they indicate 24675523Syc148097 * back pressure rather than HW failures. 24685523Syc148097 */ 24695165Syc148097 case RCR_L2_ERROR: 24705165Syc148097 rdc_stats->l2_err++; 24715165Syc148097 if (rdc_stats->l2_err < 24725165Syc148097 error_disp_cnt) { 24735165Syc148097 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 24745165Syc148097 " nxge_receive_packet:" 24755165Syc148097 " channel %d RCR L2_ERROR", 24765165Syc148097 channel)); 24775165Syc148097 } 24785165Syc148097 break; 24795165Syc148097 case RCR_L4_CSUM_ERROR: 24805165Syc148097 error_send_up = B_TRUE; 24815165Syc148097 rdc_stats->l4_cksum_err++; 24825165Syc148097 if (rdc_stats->l4_cksum_err < 24835165Syc148097 error_disp_cnt) { 24843859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 24855165Syc148097 " nxge_receive_packet:" 24865165Syc148097 " channel %d" 24875165Syc148097 " RCR L4_CSUM_ERROR", channel)); 24885165Syc148097 } 24895165Syc148097 break; 24905523Syc148097 /* 24915523Syc148097 * Do not send FMA ereport for RCR_FFLP_SOFT_ERROR and 24925523Syc148097 * RCR_ZCP_SOFT_ERROR because they reflect the same 24935523Syc148097 * FFLP and ZCP errors that have been reported by 24945523Syc148097 * nxge_fflp.c and nxge_zcp.c. 24955523Syc148097 */ 24965165Syc148097 case RCR_FFLP_SOFT_ERROR: 24975165Syc148097 error_send_up = B_TRUE; 24985165Syc148097 rdc_stats->fflp_soft_err++; 24995165Syc148097 if (rdc_stats->fflp_soft_err < 25005165Syc148097 error_disp_cnt) { 25015165Syc148097 NXGE_ERROR_MSG((nxgep, 25025165Syc148097 NXGE_ERR_CTL, 25035165Syc148097 " nxge_receive_packet:" 25045165Syc148097 " channel %d" 25055165Syc148097 " RCR FFLP_SOFT_ERROR", channel)); 25065165Syc148097 } 25075165Syc148097 break; 25085165Syc148097 case RCR_ZCP_SOFT_ERROR: 25095165Syc148097 error_send_up = B_TRUE; 25105165Syc148097 rdc_stats->fflp_soft_err++; 25115165Syc148097 if (rdc_stats->zcp_soft_err < 25125165Syc148097 error_disp_cnt) 25135165Syc148097 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 25145165Syc148097 " nxge_receive_packet: Channel %d" 25155165Syc148097 " RCR ZCP_SOFT_ERROR", channel)); 25165165Syc148097 break; 25175165Syc148097 default: 25185165Syc148097 rdc_stats->rcr_unknown_err++; 25195165Syc148097 if (rdc_stats->rcr_unknown_err 25205165Syc148097 < error_disp_cnt) { 25215165Syc148097 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 25225165Syc148097 " nxge_receive_packet: Channel %d" 25235165Syc148097 " RCR entry 0x%llx error 0x%x", 25245165Syc148097 rcr_entry, channel, error_type)); 25255165Syc148097 } 25265165Syc148097 break; 25273859Sml29623 } 25283859Sml29623 } 25293859Sml29623 25303859Sml29623 /* 25313859Sml29623 * Update and repost buffer block if max usage 25323859Sml29623 * count is reached. 25333859Sml29623 */ 25343859Sml29623 if (error_send_up == B_FALSE) { 25354874Sml29623 atomic_inc_32(&rx_msg_p->ref_cnt); 25363859Sml29623 if (buffer_free == B_TRUE) { 25373859Sml29623 rx_msg_p->free = B_TRUE; 25383859Sml29623 } 25393859Sml29623 25403859Sml29623 MUTEX_EXIT(&rx_rbr_p->lock); 25413859Sml29623 MUTEX_EXIT(&rcr_p->lock); 25423859Sml29623 nxge_freeb(rx_msg_p); 25433859Sml29623 return; 25443859Sml29623 } 25453859Sml29623 } 25463859Sml29623 25473859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 25486929Smisaki "==> nxge_receive_packet: DMA sync second ")); 25493859Sml29623 25505165Syc148097 bytes_read = rcr_p->rcvd_pkt_bytes; 25513859Sml29623 skip_len = sw_offset_bytes + hdr_size; 25523859Sml29623 if (!rx_msg_p->rx_use_bcopy) { 25534874Sml29623 /* 25544874Sml29623 * For loaned up buffers, the driver reference count 25554874Sml29623 * will be incremented first and then the free state. 25564874Sml29623 */ 25575165Syc148097 if ((nmp = nxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) { 25585165Syc148097 if (first_entry) { 25595165Syc148097 nmp->b_rptr = &nmp->b_rptr[skip_len]; 25605165Syc148097 if (l2_len < bsize - skip_len) { 25615165Syc148097 nmp->b_wptr = &nmp->b_rptr[l2_len]; 25625165Syc148097 } else { 25635165Syc148097 nmp->b_wptr = &nmp->b_rptr[bsize 25645165Syc148097 - skip_len]; 25655165Syc148097 } 25665165Syc148097 } else { 25675165Syc148097 if (l2_len - bytes_read < bsize) { 25685165Syc148097 nmp->b_wptr = 25695165Syc148097 &nmp->b_rptr[l2_len - bytes_read]; 25705165Syc148097 } else { 25715165Syc148097 nmp->b_wptr = &nmp->b_rptr[bsize]; 25725165Syc148097 } 25735165Syc148097 } 25745165Syc148097 } 25753859Sml29623 } else { 25765165Syc148097 if (first_entry) { 25775165Syc148097 nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len, 25785165Syc148097 l2_len < bsize - skip_len ? 25795165Syc148097 l2_len : bsize - skip_len); 25805165Syc148097 } else { 25815165Syc148097 nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset, 25825165Syc148097 l2_len - bytes_read < bsize ? 25835165Syc148097 l2_len - bytes_read : bsize); 25845165Syc148097 } 25853859Sml29623 } 25863859Sml29623 if (nmp != NULL) { 25877145Syc148097 if (first_entry) { 25887145Syc148097 /* 25897145Syc148097 * Jumbo packets may be received with more than one 25907145Syc148097 * buffer, increment ipackets for the first entry only. 25917145Syc148097 */ 25927145Syc148097 rdc_stats->ipackets++; 25937145Syc148097 25947145Syc148097 /* Update ibytes for kstat. */ 25957145Syc148097 rdc_stats->ibytes += skip_len 25967145Syc148097 + l2_len < bsize ? l2_len : bsize; 25977145Syc148097 /* 25987145Syc148097 * Update the number of bytes read so far for the 25997145Syc148097 * current frame. 26007145Syc148097 */ 26015165Syc148097 bytes_read = nmp->b_wptr - nmp->b_rptr; 26027145Syc148097 } else { 26037145Syc148097 rdc_stats->ibytes += l2_len - bytes_read < bsize ? 26047145Syc148097 l2_len - bytes_read : bsize; 26053859Sml29623 bytes_read += nmp->b_wptr - nmp->b_rptr; 26067145Syc148097 } 26075165Syc148097 26085165Syc148097 NXGE_DEBUG_MSG((nxgep, RX_CTL, 26095165Syc148097 "==> nxge_receive_packet after dupb: " 26105165Syc148097 "rbr consumed %d " 26115165Syc148097 "pktbufsz_type %d " 26125165Syc148097 "nmp $%p rptr $%p wptr $%p " 26135165Syc148097 "buf_offset %d bzise %d l2_len %d skip_len %d", 26145165Syc148097 rx_rbr_p->rbr_consumed, 26155165Syc148097 pktbufsz_type, 26165165Syc148097 nmp, nmp->b_rptr, nmp->b_wptr, 26175165Syc148097 buf_offset, bsize, l2_len, skip_len)); 26183859Sml29623 } else { 26193859Sml29623 cmn_err(CE_WARN, "!nxge_receive_packet: " 26206929Smisaki "update stats (error)"); 26214977Sraghus atomic_inc_32(&rx_msg_p->ref_cnt); 26224977Sraghus if (buffer_free == B_TRUE) { 26234977Sraghus rx_msg_p->free = B_TRUE; 26244977Sraghus } 26254977Sraghus MUTEX_EXIT(&rx_rbr_p->lock); 26264977Sraghus MUTEX_EXIT(&rcr_p->lock); 26274977Sraghus nxge_freeb(rx_msg_p); 26284977Sraghus return; 26293859Sml29623 } 26305060Syc148097 26313859Sml29623 if (buffer_free == B_TRUE) { 26323859Sml29623 rx_msg_p->free = B_TRUE; 26333859Sml29623 } 26347145Syc148097 26353859Sml29623 is_valid = (nmp != NULL); 26365165Syc148097 26375165Syc148097 rcr_p->rcvd_pkt_bytes = bytes_read; 26385165Syc148097 26393859Sml29623 MUTEX_EXIT(&rx_rbr_p->lock); 26403859Sml29623 MUTEX_EXIT(&rcr_p->lock); 26413859Sml29623 26423859Sml29623 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) { 26433859Sml29623 atomic_inc_32(&rx_msg_p->ref_cnt); 26443859Sml29623 nxge_freeb(rx_msg_p); 26453859Sml29623 } 26463859Sml29623 26473859Sml29623 if (is_valid) { 26483859Sml29623 nmp->b_cont = NULL; 26493859Sml29623 if (first_entry) { 26503859Sml29623 *mp = nmp; 26513859Sml29623 *mp_cont = NULL; 26525165Syc148097 } else { 26533859Sml29623 *mp_cont = nmp; 26545165Syc148097 } 26553859Sml29623 } 26563859Sml29623 26573859Sml29623 /* 26587145Syc148097 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. 26597145Syc148097 * If a packet is not fragmented and no error bit is set, then 26607145Syc148097 * L4 checksum is OK. 26613859Sml29623 */ 26627145Syc148097 26633859Sml29623 if (is_valid && !multi) { 26646495Sspeer /* 26657145Syc148097 * Update hardware checksuming. 26667145Syc148097 * 26676611Sml29623 * If the checksum flag nxge_chksum_offload 26686611Sml29623 * is 1, TCP and UDP packets can be sent 26696495Sspeer * up with good checksum. If the checksum flag 26706611Sml29623 * is set to 0, checksum reporting will apply to 26716495Sspeer * TCP packets only (workaround for a hardware bug). 26726611Sml29623 * If the checksum flag nxge_cksum_offload is 26736611Sml29623 * greater than 1, both TCP and UDP packets 26746611Sml29623 * will not be reported its hardware checksum results. 26756495Sspeer */ 26766611Sml29623 if (nxge_cksum_offload == 1) { 26776495Sspeer is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP || 26786929Smisaki pkt_type == RCR_PKT_IS_UDP) ? 26796929Smisaki B_TRUE: B_FALSE); 26806611Sml29623 } else if (!nxge_cksum_offload) { 26816495Sspeer /* TCP checksum only. */ 26826495Sspeer is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP) ? 26836929Smisaki B_TRUE: B_FALSE); 26846495Sspeer } 26853859Sml29623 26863859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_receive_packet: " 26876929Smisaki "is_valid 0x%x multi 0x%llx pkt %d frag %d error %d", 26886929Smisaki is_valid, multi, is_tcp_udp, frag, error_type)); 26893859Sml29623 26903859Sml29623 if (is_tcp_udp && !frag && !error_type) { 26913859Sml29623 (void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0, 26926929Smisaki HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0); 26933859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 26946929Smisaki "==> nxge_receive_packet: Full tcp/udp cksum " 26956929Smisaki "is_valid 0x%x multi 0x%llx pkt %d frag %d " 26966929Smisaki "error %d", 26976929Smisaki is_valid, multi, is_tcp_udp, frag, error_type)); 26983859Sml29623 } 26993859Sml29623 } 27003859Sml29623 27013859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, 27026929Smisaki "==> nxge_receive_packet: *mp 0x%016llx", *mp)); 27033859Sml29623 27043859Sml29623 *multi_p = (multi == RCR_MULTI_MASK); 27053859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_receive_packet: " 27066929Smisaki "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx", 27076929Smisaki *multi_p, nmp, *mp, *mp_cont)); 27083859Sml29623 } 27093859Sml29623 27103859Sml29623 /*ARGSUSED*/ 27113859Sml29623 static nxge_status_t 27126495Sspeer nxge_rx_err_evnts(p_nxge_t nxgep, int channel, rx_dma_ctl_stat_t cs) 27133859Sml29623 { 27143859Sml29623 p_nxge_rx_ring_stats_t rdc_stats; 27153859Sml29623 npi_handle_t handle; 27163859Sml29623 npi_status_t rs; 27173859Sml29623 boolean_t rxchan_fatal = B_FALSE; 27183859Sml29623 boolean_t rxport_fatal = B_FALSE; 27193859Sml29623 uint8_t portn; 27203859Sml29623 nxge_status_t status = NXGE_OK; 27213859Sml29623 uint32_t error_disp_cnt = NXGE_ERROR_SHOW_MAX; 27223859Sml29623 NXGE_DEBUG_MSG((nxgep, INT_CTL, "==> nxge_rx_err_evnts")); 27233859Sml29623 27243859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 27253859Sml29623 portn = nxgep->mac.portnum; 27266495Sspeer rdc_stats = &nxgep->statsp->rdc_stats[channel]; 27273859Sml29623 27283859Sml29623 if (cs.bits.hdw.rbr_tmout) { 27293859Sml29623 rdc_stats->rx_rbr_tmout++; 27303859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 27316929Smisaki NXGE_FM_EREPORT_RDMC_RBR_TMOUT); 27323859Sml29623 rxchan_fatal = B_TRUE; 27333859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27346929Smisaki "==> nxge_rx_err_evnts: rx_rbr_timeout")); 27353859Sml29623 } 27363859Sml29623 if (cs.bits.hdw.rsp_cnt_err) { 27373859Sml29623 rdc_stats->rsp_cnt_err++; 27383859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 27396929Smisaki NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR); 27403859Sml29623 rxchan_fatal = B_TRUE; 27413859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27426929Smisaki "==> nxge_rx_err_evnts(channel %d): " 27436929Smisaki "rsp_cnt_err", channel)); 27443859Sml29623 } 27453859Sml29623 if (cs.bits.hdw.byte_en_bus) { 27463859Sml29623 rdc_stats->byte_en_bus++; 27473859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 27486929Smisaki NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS); 27493859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27506929Smisaki "==> nxge_rx_err_evnts(channel %d): " 27516929Smisaki "fatal error: byte_en_bus", channel)); 27523859Sml29623 rxchan_fatal = B_TRUE; 27533859Sml29623 } 27543859Sml29623 if (cs.bits.hdw.rsp_dat_err) { 27553859Sml29623 rdc_stats->rsp_dat_err++; 27563859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 27576929Smisaki NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR); 27583859Sml29623 rxchan_fatal = B_TRUE; 27593859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27606929Smisaki "==> nxge_rx_err_evnts(channel %d): " 27616929Smisaki "fatal error: rsp_dat_err", channel)); 27623859Sml29623 } 27633859Sml29623 if (cs.bits.hdw.rcr_ack_err) { 27643859Sml29623 rdc_stats->rcr_ack_err++; 27653859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 27666929Smisaki NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR); 27673859Sml29623 rxchan_fatal = B_TRUE; 27683859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27696929Smisaki "==> nxge_rx_err_evnts(channel %d): " 27706929Smisaki "fatal error: rcr_ack_err", channel)); 27713859Sml29623 } 27723859Sml29623 if (cs.bits.hdw.dc_fifo_err) { 27733859Sml29623 rdc_stats->dc_fifo_err++; 27743859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 27756929Smisaki NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR); 27763859Sml29623 /* This is not a fatal error! */ 27773859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27786929Smisaki "==> nxge_rx_err_evnts(channel %d): " 27796929Smisaki "dc_fifo_err", channel)); 27803859Sml29623 rxport_fatal = B_TRUE; 27813859Sml29623 } 27823859Sml29623 if ((cs.bits.hdw.rcr_sha_par) || (cs.bits.hdw.rbr_pre_par)) { 27833859Sml29623 if ((rs = npi_rxdma_ring_perr_stat_get(handle, 27846929Smisaki &rdc_stats->errlog.pre_par, 27856929Smisaki &rdc_stats->errlog.sha_par)) 27866929Smisaki != NPI_SUCCESS) { 27873859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27886929Smisaki "==> nxge_rx_err_evnts(channel %d): " 27896929Smisaki "rcr_sha_par: get perr", channel)); 27903859Sml29623 return (NXGE_ERROR | rs); 27913859Sml29623 } 27923859Sml29623 if (cs.bits.hdw.rcr_sha_par) { 27933859Sml29623 rdc_stats->rcr_sha_par++; 27943859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 27956929Smisaki NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR); 27963859Sml29623 rxchan_fatal = B_TRUE; 27973859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 27986929Smisaki "==> nxge_rx_err_evnts(channel %d): " 27996929Smisaki "fatal error: rcr_sha_par", channel)); 28003859Sml29623 } 28013859Sml29623 if (cs.bits.hdw.rbr_pre_par) { 28023859Sml29623 rdc_stats->rbr_pre_par++; 28033859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 28046929Smisaki NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR); 28053859Sml29623 rxchan_fatal = B_TRUE; 28063859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28076929Smisaki "==> nxge_rx_err_evnts(channel %d): " 28086929Smisaki "fatal error: rbr_pre_par", channel)); 28093859Sml29623 } 28103859Sml29623 } 28116172Syc148097 /* 28126172Syc148097 * The Following 4 status bits are for information, the system 28136172Syc148097 * is running fine. There is no need to send FMA ereports or 28146172Syc148097 * log messages. 28156172Syc148097 */ 28163859Sml29623 if (cs.bits.hdw.port_drop_pkt) { 28173859Sml29623 rdc_stats->port_drop_pkt++; 28183859Sml29623 } 28193859Sml29623 if (cs.bits.hdw.wred_drop) { 28203859Sml29623 rdc_stats->wred_drop++; 28213859Sml29623 } 28223859Sml29623 if (cs.bits.hdw.rbr_pre_empty) { 28233859Sml29623 rdc_stats->rbr_pre_empty++; 28243859Sml29623 } 28253859Sml29623 if (cs.bits.hdw.rcr_shadow_full) { 28263859Sml29623 rdc_stats->rcr_shadow_full++; 28273859Sml29623 } 28283859Sml29623 if (cs.bits.hdw.config_err) { 28293859Sml29623 rdc_stats->config_err++; 28303859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 28316929Smisaki NXGE_FM_EREPORT_RDMC_CONFIG_ERR); 28323859Sml29623 rxchan_fatal = B_TRUE; 28333859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28346929Smisaki "==> nxge_rx_err_evnts(channel %d): " 28356929Smisaki "config error", channel)); 28363859Sml29623 } 28373859Sml29623 if (cs.bits.hdw.rcrincon) { 28383859Sml29623 rdc_stats->rcrincon++; 28393859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 28406929Smisaki NXGE_FM_EREPORT_RDMC_RCRINCON); 28413859Sml29623 rxchan_fatal = B_TRUE; 28423859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28436929Smisaki "==> nxge_rx_err_evnts(channel %d): " 28446929Smisaki "fatal error: rcrincon error", channel)); 28453859Sml29623 } 28463859Sml29623 if (cs.bits.hdw.rcrfull) { 28473859Sml29623 rdc_stats->rcrfull++; 28483859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 28496929Smisaki NXGE_FM_EREPORT_RDMC_RCRFULL); 28503859Sml29623 rxchan_fatal = B_TRUE; 28513859Sml29623 if (rdc_stats->rcrfull < error_disp_cnt) 28523859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28536929Smisaki "==> nxge_rx_err_evnts(channel %d): " 28546929Smisaki "fatal error: rcrfull error", channel)); 28553859Sml29623 } 28563859Sml29623 if (cs.bits.hdw.rbr_empty) { 28576172Syc148097 /* 28586172Syc148097 * This bit is for information, there is no need 28596172Syc148097 * send FMA ereport or log a message. 28606172Syc148097 */ 28613859Sml29623 rdc_stats->rbr_empty++; 28623859Sml29623 } 28633859Sml29623 if (cs.bits.hdw.rbrfull) { 28643859Sml29623 rdc_stats->rbrfull++; 28653859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 28666929Smisaki NXGE_FM_EREPORT_RDMC_RBRFULL); 28673859Sml29623 rxchan_fatal = B_TRUE; 28683859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28696929Smisaki "==> nxge_rx_err_evnts(channel %d): " 28706929Smisaki "fatal error: rbr_full error", channel)); 28713859Sml29623 } 28723859Sml29623 if (cs.bits.hdw.rbrlogpage) { 28733859Sml29623 rdc_stats->rbrlogpage++; 28743859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 28756929Smisaki NXGE_FM_EREPORT_RDMC_RBRLOGPAGE); 28763859Sml29623 rxchan_fatal = B_TRUE; 28773859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28786929Smisaki "==> nxge_rx_err_evnts(channel %d): " 28796929Smisaki "fatal error: rbr logical page error", channel)); 28803859Sml29623 } 28813859Sml29623 if (cs.bits.hdw.cfiglogpage) { 28823859Sml29623 rdc_stats->cfiglogpage++; 28833859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, channel, 28846929Smisaki NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE); 28853859Sml29623 rxchan_fatal = B_TRUE; 28863859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28876929Smisaki "==> nxge_rx_err_evnts(channel %d): " 28886929Smisaki "fatal error: cfig logical page error", channel)); 28893859Sml29623 } 28903859Sml29623 28913859Sml29623 if (rxport_fatal) { 28923859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 28936495Sspeer " nxge_rx_err_evnts: fatal error on Port #%d\n", 28946495Sspeer portn)); 28956495Sspeer if (isLDOMguest(nxgep)) { 28966495Sspeer status = NXGE_ERROR; 28976495Sspeer } else { 28986495Sspeer status = nxge_ipp_fatal_err_recover(nxgep); 28996495Sspeer if (status == NXGE_OK) { 29006495Sspeer FM_SERVICE_RESTORED(nxgep); 29016495Sspeer } 29023859Sml29623 } 29033859Sml29623 } 29043859Sml29623 29053859Sml29623 if (rxchan_fatal) { 29063859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 29076495Sspeer " nxge_rx_err_evnts: fatal error on Channel #%d\n", 29086495Sspeer channel)); 29096495Sspeer if (isLDOMguest(nxgep)) { 29106495Sspeer status = NXGE_ERROR; 29116495Sspeer } else { 29126495Sspeer status = nxge_rxdma_fatal_err_recover(nxgep, channel); 29136495Sspeer if (status == NXGE_OK) { 29146495Sspeer FM_SERVICE_RESTORED(nxgep); 29156495Sspeer } 29163859Sml29623 } 29173859Sml29623 } 29183859Sml29623 29193859Sml29623 NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rx_err_evnts")); 29203859Sml29623 29213859Sml29623 return (status); 29223859Sml29623 } 29233859Sml29623 29246495Sspeer /* 29256495Sspeer * nxge_rdc_hvio_setup 29266495Sspeer * 29276495Sspeer * This code appears to setup some Hypervisor variables. 29286495Sspeer * 29296495Sspeer * Arguments: 29306495Sspeer * nxgep 29316495Sspeer * channel 29326495Sspeer * 29336495Sspeer * Notes: 29346495Sspeer * What does NIU_LP_WORKAROUND mean? 29356495Sspeer * 29366495Sspeer * NPI/NXGE function calls: 29376495Sspeer * na 29386495Sspeer * 29396495Sspeer * Context: 29406495Sspeer * Any domain 29416495Sspeer */ 29426495Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 29436495Sspeer static void 29446495Sspeer nxge_rdc_hvio_setup( 29456495Sspeer nxge_t *nxgep, int channel) 29463859Sml29623 { 29476495Sspeer nxge_dma_common_t *dma_common; 29486495Sspeer nxge_dma_common_t *dma_control; 29496495Sspeer rx_rbr_ring_t *ring; 29506495Sspeer 29516495Sspeer ring = nxgep->rx_rbr_rings->rbr_rings[channel]; 29526495Sspeer dma_common = nxgep->rx_buf_pool_p->dma_buf_pool_p[channel]; 29536495Sspeer 29546495Sspeer ring->hv_set = B_FALSE; 29556495Sspeer 29566495Sspeer ring->hv_rx_buf_base_ioaddr_pp = (uint64_t) 29576495Sspeer dma_common->orig_ioaddr_pp; 29586495Sspeer ring->hv_rx_buf_ioaddr_size = (uint64_t) 29596495Sspeer dma_common->orig_alength; 29606495Sspeer 29616495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: " 29626495Sspeer "channel %d data buf base io $%lx ($%p) size 0x%lx (%ld 0x%lx)", 29636495Sspeer channel, ring->hv_rx_buf_base_ioaddr_pp, 29646495Sspeer dma_common->ioaddr_pp, ring->hv_rx_buf_ioaddr_size, 29656495Sspeer dma_common->orig_alength, dma_common->orig_alength)); 29666495Sspeer 29676495Sspeer dma_control = nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel]; 29686495Sspeer 29696495Sspeer ring->hv_rx_cntl_base_ioaddr_pp = 29706495Sspeer (uint64_t)dma_control->orig_ioaddr_pp; 29716495Sspeer ring->hv_rx_cntl_ioaddr_size = 29726495Sspeer (uint64_t)dma_control->orig_alength; 29736495Sspeer 29746495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: " 29756495Sspeer "channel %d cntl base io $%p ($%p) size 0x%llx (%d 0x%x)", 29766495Sspeer channel, ring->hv_rx_cntl_base_ioaddr_pp, 29776495Sspeer dma_control->ioaddr_pp, ring->hv_rx_cntl_ioaddr_size, 29786495Sspeer dma_control->orig_alength, dma_control->orig_alength)); 29796495Sspeer } 29803859Sml29623 #endif 29813859Sml29623 29826495Sspeer /* 29836495Sspeer * nxge_map_rxdma 29846495Sspeer * 29856495Sspeer * Map an RDC into our kernel space. 29866495Sspeer * 29876495Sspeer * Arguments: 29886495Sspeer * nxgep 29896495Sspeer * channel The channel to map. 29906495Sspeer * 29916495Sspeer * Notes: 29926495Sspeer * 1. Allocate & initialise a memory pool, if necessary. 29936495Sspeer * 2. Allocate however many receive buffers are required. 29946495Sspeer * 3. Setup buffers, descriptors, and mailbox. 29956495Sspeer * 29966495Sspeer * NPI/NXGE function calls: 29976495Sspeer * nxge_alloc_rx_mem_pool() 29986495Sspeer * nxge_alloc_rbb() 29996495Sspeer * nxge_map_rxdma_channel() 30006495Sspeer * 30016495Sspeer * Registers accessed: 30026495Sspeer * 30036495Sspeer * Context: 30046495Sspeer * Any domain 30056495Sspeer */ 30066495Sspeer static nxge_status_t 30076495Sspeer nxge_map_rxdma(p_nxge_t nxgep, int channel) 30086495Sspeer { 30096495Sspeer nxge_dma_common_t **data; 30106495Sspeer nxge_dma_common_t **control; 30116495Sspeer rx_rbr_ring_t **rbr_ring; 30126495Sspeer rx_rcr_ring_t **rcr_ring; 30136495Sspeer rx_mbox_t **mailbox; 30146495Sspeer uint32_t chunks; 30156495Sspeer 30166495Sspeer nxge_status_t status; 30176495Sspeer 30183859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma")); 30193859Sml29623 30206495Sspeer if (!nxgep->rx_buf_pool_p) { 30216495Sspeer if (nxge_alloc_rx_mem_pool(nxgep) != NXGE_OK) { 30226495Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 30236495Sspeer "<== nxge_map_rxdma: buf not allocated")); 30246495Sspeer return (NXGE_ERROR); 30256495Sspeer } 30263859Sml29623 } 30273859Sml29623 30286495Sspeer if (nxge_alloc_rxb(nxgep, channel) != NXGE_OK) 30296495Sspeer return (NXGE_ERROR); 30303859Sml29623 30313859Sml29623 /* 30323859Sml29623 * Timeout should be set based on the system clock divider. 30333859Sml29623 * The following timeout value of 1 assumes that the 30343859Sml29623 * granularity (1000) is 3 microseconds running at 300MHz. 30353859Sml29623 */ 30363859Sml29623 30373859Sml29623 nxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT; 30383859Sml29623 nxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT; 30393859Sml29623 30403859Sml29623 /* 30416495Sspeer * Map descriptors from the buffer polls for each dma channel. 30426495Sspeer */ 30436495Sspeer 30446495Sspeer /* 30456495Sspeer * Set up and prepare buffer blocks, descriptors 30466495Sspeer * and mailbox. 30473859Sml29623 */ 30486495Sspeer data = &nxgep->rx_buf_pool_p->dma_buf_pool_p[channel]; 30496495Sspeer rbr_ring = &nxgep->rx_rbr_rings->rbr_rings[channel]; 30506495Sspeer chunks = nxgep->rx_buf_pool_p->num_chunks[channel]; 30516495Sspeer 30526495Sspeer control = &nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel]; 30536495Sspeer rcr_ring = &nxgep->rx_rcr_rings->rcr_rings[channel]; 30546495Sspeer 30556495Sspeer mailbox = &nxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 30566495Sspeer 30576495Sspeer status = nxge_map_rxdma_channel(nxgep, channel, data, rbr_ring, 30586495Sspeer chunks, control, rcr_ring, mailbox); 30596495Sspeer if (status != NXGE_OK) { 30606495Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 30616929Smisaki "==> nxge_map_rxdma: nxge_map_rxdma_channel(%d) " 30626929Smisaki "returned 0x%x", 30636929Smisaki channel, status)); 30646495Sspeer return (status); 30656495Sspeer } 30666495Sspeer nxgep->rx_rbr_rings->rbr_rings[channel]->index = (uint16_t)channel; 30676495Sspeer nxgep->rx_rcr_rings->rcr_rings[channel]->index = (uint16_t)channel; 30686495Sspeer nxgep->rx_rcr_rings->rcr_rings[channel]->rdc_stats = 30696495Sspeer &nxgep->statsp->rdc_stats[channel]; 30703859Sml29623 30713859Sml29623 #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 30726495Sspeer if (!isLDOMguest(nxgep)) 30736495Sspeer nxge_rdc_hvio_setup(nxgep, channel); 30746495Sspeer #endif 30756495Sspeer 30763859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 30776495Sspeer "<== nxge_map_rxdma: (status 0x%x channel %d)", status, channel)); 30783859Sml29623 30793859Sml29623 return (status); 30803859Sml29623 } 30813859Sml29623 30823859Sml29623 static void 30836495Sspeer nxge_unmap_rxdma(p_nxge_t nxgep, int channel) 30843859Sml29623 { 30856495Sspeer rx_rbr_ring_t *rbr_ring; 30866495Sspeer rx_rcr_ring_t *rcr_ring; 30876495Sspeer rx_mbox_t *mailbox; 30886495Sspeer 30896495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_unmap_rxdma(%d)", channel)); 30906495Sspeer 30916495Sspeer if (!nxgep->rx_rbr_rings || !nxgep->rx_rcr_rings || 30926495Sspeer !nxgep->rx_mbox_areas_p) 30933859Sml29623 return; 30946495Sspeer 30956495Sspeer rbr_ring = nxgep->rx_rbr_rings->rbr_rings[channel]; 30966495Sspeer rcr_ring = nxgep->rx_rcr_rings->rcr_rings[channel]; 30976495Sspeer mailbox = nxgep->rx_mbox_areas_p->rxmbox_areas[channel]; 30986495Sspeer 30996495Sspeer if (!rbr_ring || !rcr_ring || !mailbox) 31003859Sml29623 return; 31016495Sspeer 31026495Sspeer (void) nxge_unmap_rxdma_channel( 31036929Smisaki nxgep, channel, rbr_ring, rcr_ring, mailbox); 31046495Sspeer 31056495Sspeer nxge_free_rxb(nxgep, channel); 31066495Sspeer 31076495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma")); 31083859Sml29623 } 31093859Sml29623 31103859Sml29623 nxge_status_t 31113859Sml29623 nxge_map_rxdma_channel(p_nxge_t nxgep, uint16_t channel, 31123859Sml29623 p_nxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p, 31133859Sml29623 uint32_t num_chunks, 31143859Sml29623 p_nxge_dma_common_t *dma_cntl_p, p_rx_rcr_ring_t *rcr_p, 31153859Sml29623 p_rx_mbox_t *rx_mbox_p) 31163859Sml29623 { 31173859Sml29623 int status = NXGE_OK; 31183859Sml29623 31193859Sml29623 /* 31203859Sml29623 * Set up and prepare buffer blocks, descriptors 31213859Sml29623 * and mailbox. 31223859Sml29623 */ 31233859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 31246929Smisaki "==> nxge_map_rxdma_channel (channel %d)", channel)); 31253859Sml29623 /* 31263859Sml29623 * Receive buffer blocks 31273859Sml29623 */ 31283859Sml29623 status = nxge_map_rxdma_channel_buf_ring(nxgep, channel, 31296929Smisaki dma_buf_p, rbr_p, num_chunks); 31303859Sml29623 if (status != NXGE_OK) { 31313859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 31326929Smisaki "==> nxge_map_rxdma_channel (channel %d): " 31336929Smisaki "map buffer failed 0x%x", channel, status)); 31343859Sml29623 goto nxge_map_rxdma_channel_exit; 31353859Sml29623 } 31363859Sml29623 31373859Sml29623 /* 31383859Sml29623 * Receive block ring, completion ring and mailbox. 31393859Sml29623 */ 31403859Sml29623 status = nxge_map_rxdma_channel_cfg_ring(nxgep, channel, 31416929Smisaki dma_cntl_p, rbr_p, rcr_p, rx_mbox_p); 31423859Sml29623 if (status != NXGE_OK) { 31433859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 31446929Smisaki "==> nxge_map_rxdma_channel (channel %d): " 31456929Smisaki "map config failed 0x%x", channel, status)); 31463859Sml29623 goto nxge_map_rxdma_channel_fail2; 31473859Sml29623 } 31483859Sml29623 31493859Sml29623 goto nxge_map_rxdma_channel_exit; 31503859Sml29623 31513859Sml29623 nxge_map_rxdma_channel_fail3: 31523859Sml29623 /* Free rbr, rcr */ 31533859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 31546929Smisaki "==> nxge_map_rxdma_channel: free rbr/rcr " 31556929Smisaki "(status 0x%x channel %d)", 31566929Smisaki status, channel)); 31573859Sml29623 nxge_unmap_rxdma_channel_cfg_ring(nxgep, 31586929Smisaki *rcr_p, *rx_mbox_p); 31593859Sml29623 31603859Sml29623 nxge_map_rxdma_channel_fail2: 31613859Sml29623 /* Free buffer blocks */ 31623859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 31636929Smisaki "==> nxge_map_rxdma_channel: free rx buffers" 31646929Smisaki "(nxgep 0x%x status 0x%x channel %d)", 31656929Smisaki nxgep, status, channel)); 31663859Sml29623 nxge_unmap_rxdma_channel_buf_ring(nxgep, *rbr_p); 31673859Sml29623 31684185Sspeer status = NXGE_ERROR; 31694185Sspeer 31703859Sml29623 nxge_map_rxdma_channel_exit: 31713859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 31726929Smisaki "<== nxge_map_rxdma_channel: " 31736929Smisaki "(nxgep 0x%x status 0x%x channel %d)", 31746929Smisaki nxgep, status, channel)); 31753859Sml29623 31763859Sml29623 return (status); 31773859Sml29623 } 31783859Sml29623 31793859Sml29623 /*ARGSUSED*/ 31803859Sml29623 static void 31813859Sml29623 nxge_unmap_rxdma_channel(p_nxge_t nxgep, uint16_t channel, 31823859Sml29623 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 31833859Sml29623 { 31843859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 31856929Smisaki "==> nxge_unmap_rxdma_channel (channel %d)", channel)); 31863859Sml29623 31873859Sml29623 /* 31883859Sml29623 * unmap receive block ring, completion ring and mailbox. 31893859Sml29623 */ 31903859Sml29623 (void) nxge_unmap_rxdma_channel_cfg_ring(nxgep, 31916929Smisaki rcr_p, rx_mbox_p); 31923859Sml29623 31933859Sml29623 /* unmap buffer blocks */ 31943859Sml29623 (void) nxge_unmap_rxdma_channel_buf_ring(nxgep, rbr_p); 31953859Sml29623 31963859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma_channel")); 31973859Sml29623 } 31983859Sml29623 31993859Sml29623 /*ARGSUSED*/ 32003859Sml29623 static nxge_status_t 32013859Sml29623 nxge_map_rxdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel, 32023859Sml29623 p_nxge_dma_common_t *dma_cntl_p, p_rx_rbr_ring_t *rbr_p, 32033859Sml29623 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p) 32043859Sml29623 { 32053859Sml29623 p_rx_rbr_ring_t rbrp; 32063859Sml29623 p_rx_rcr_ring_t rcrp; 32073859Sml29623 p_rx_mbox_t mboxp; 32083859Sml29623 p_nxge_dma_common_t cntl_dmap; 32093859Sml29623 p_nxge_dma_common_t dmap; 32103859Sml29623 p_rx_msg_t *rx_msg_ring; 32113859Sml29623 p_rx_msg_t rx_msg_p; 32123859Sml29623 p_rbr_cfig_a_t rcfga_p; 32133859Sml29623 p_rbr_cfig_b_t rcfgb_p; 32143859Sml29623 p_rcrcfig_a_t cfga_p; 32153859Sml29623 p_rcrcfig_b_t cfgb_p; 32163859Sml29623 p_rxdma_cfig1_t cfig1_p; 32173859Sml29623 p_rxdma_cfig2_t cfig2_p; 32183859Sml29623 p_rbr_kick_t kick_p; 32193859Sml29623 uint32_t dmaaddrp; 32203859Sml29623 uint32_t *rbr_vaddrp; 32213859Sml29623 uint32_t bkaddr; 32223859Sml29623 nxge_status_t status = NXGE_OK; 32233859Sml29623 int i; 32243859Sml29623 uint32_t nxge_port_rcr_size; 32253859Sml29623 32263859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 32276929Smisaki "==> nxge_map_rxdma_channel_cfg_ring")); 32283859Sml29623 32293859Sml29623 cntl_dmap = *dma_cntl_p; 32303859Sml29623 32313859Sml29623 /* Map in the receive block ring */ 32323859Sml29623 rbrp = *rbr_p; 32333859Sml29623 dmap = (p_nxge_dma_common_t)&rbrp->rbr_desc; 32343859Sml29623 nxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4); 32353859Sml29623 /* 32363859Sml29623 * Zero out buffer block ring descriptors. 32373859Sml29623 */ 32383859Sml29623 bzero((caddr_t)dmap->kaddrp, dmap->alength); 32393859Sml29623 32403859Sml29623 rcfga_p = &(rbrp->rbr_cfga); 32413859Sml29623 rcfgb_p = &(rbrp->rbr_cfgb); 32423859Sml29623 kick_p = &(rbrp->rbr_kick); 32433859Sml29623 rcfga_p->value = 0; 32443859Sml29623 rcfgb_p->value = 0; 32453859Sml29623 kick_p->value = 0; 32463859Sml29623 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress; 32473859Sml29623 rcfga_p->value = (rbrp->rbr_addr & 32486929Smisaki (RBR_CFIG_A_STDADDR_MASK | 32496929Smisaki RBR_CFIG_A_STDADDR_BASE_MASK)); 32503859Sml29623 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT); 32513859Sml29623 32523859Sml29623 rcfgb_p->bits.ldw.bufsz0 = rbrp->pkt_buf_size0; 32533859Sml29623 rcfgb_p->bits.ldw.vld0 = 1; 32543859Sml29623 rcfgb_p->bits.ldw.bufsz1 = rbrp->pkt_buf_size1; 32553859Sml29623 rcfgb_p->bits.ldw.vld1 = 1; 32563859Sml29623 rcfgb_p->bits.ldw.bufsz2 = rbrp->pkt_buf_size2; 32573859Sml29623 rcfgb_p->bits.ldw.vld2 = 1; 32583859Sml29623 rcfgb_p->bits.ldw.bksize = nxgep->rx_bksize_code; 32593859Sml29623 32603859Sml29623 /* 32613859Sml29623 * For each buffer block, enter receive block address to the ring. 32623859Sml29623 */ 32633859Sml29623 rbr_vaddrp = (uint32_t *)dmap->kaddrp; 32643859Sml29623 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp; 32653859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 32666929Smisaki "==> nxge_map_rxdma_channel_cfg_ring: channel %d " 32676929Smisaki "rbr_vaddrp $%p", dma_channel, rbr_vaddrp)); 32683859Sml29623 32693859Sml29623 rx_msg_ring = rbrp->rx_msg_ring; 32703859Sml29623 for (i = 0; i < rbrp->tnblocks; i++) { 32713859Sml29623 rx_msg_p = rx_msg_ring[i]; 32723859Sml29623 rx_msg_p->nxgep = nxgep; 32733859Sml29623 rx_msg_p->rx_rbr_p = rbrp; 32743859Sml29623 bkaddr = (uint32_t) 32756929Smisaki ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress 32766929Smisaki >> RBR_BKADDR_SHIFT)); 32773859Sml29623 rx_msg_p->free = B_FALSE; 32783859Sml29623 rx_msg_p->max_usage_cnt = 0xbaddcafe; 32793859Sml29623 32803859Sml29623 *rbr_vaddrp++ = bkaddr; 32813859Sml29623 } 32823859Sml29623 32833859Sml29623 kick_p->bits.ldw.bkadd = rbrp->rbb_max; 32843859Sml29623 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 32853859Sml29623 32863859Sml29623 rbrp->rbr_rd_index = 0; 32873859Sml29623 32883859Sml29623 rbrp->rbr_consumed = 0; 32893859Sml29623 rbrp->rbr_use_bcopy = B_TRUE; 32903859Sml29623 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0; 32913859Sml29623 /* 32923859Sml29623 * Do bcopy on packets greater than bcopy size once 32933859Sml29623 * the lo threshold is reached. 32943859Sml29623 * This lo threshold should be less than the hi threshold. 32953859Sml29623 * 32963859Sml29623 * Do bcopy on every packet once the hi threshold is reached. 32973859Sml29623 */ 32983859Sml29623 if (nxge_rx_threshold_lo >= nxge_rx_threshold_hi) { 32993859Sml29623 /* default it to use hi */ 33003859Sml29623 nxge_rx_threshold_lo = nxge_rx_threshold_hi; 33013859Sml29623 } 33023859Sml29623 33033859Sml29623 if (nxge_rx_buf_size_type > NXGE_RBR_TYPE2) { 33043859Sml29623 nxge_rx_buf_size_type = NXGE_RBR_TYPE2; 33053859Sml29623 } 33063859Sml29623 rbrp->rbr_bufsize_type = nxge_rx_buf_size_type; 33073859Sml29623 33083859Sml29623 switch (nxge_rx_threshold_hi) { 33093859Sml29623 default: 33103859Sml29623 case NXGE_RX_COPY_NONE: 33113859Sml29623 /* Do not do bcopy at all */ 33123859Sml29623 rbrp->rbr_use_bcopy = B_FALSE; 33133859Sml29623 rbrp->rbr_threshold_hi = rbrp->rbb_max; 33143859Sml29623 break; 33153859Sml29623 33163859Sml29623 case NXGE_RX_COPY_1: 33173859Sml29623 case NXGE_RX_COPY_2: 33183859Sml29623 case NXGE_RX_COPY_3: 33193859Sml29623 case NXGE_RX_COPY_4: 33203859Sml29623 case NXGE_RX_COPY_5: 33213859Sml29623 case NXGE_RX_COPY_6: 33223859Sml29623 case NXGE_RX_COPY_7: 33233859Sml29623 rbrp->rbr_threshold_hi = 33246929Smisaki rbrp->rbb_max * 33256929Smisaki (nxge_rx_threshold_hi)/NXGE_RX_BCOPY_SCALE; 33263859Sml29623 break; 33273859Sml29623 33283859Sml29623 case NXGE_RX_COPY_ALL: 33293859Sml29623 rbrp->rbr_threshold_hi = 0; 33303859Sml29623 break; 33313859Sml29623 } 33323859Sml29623 33333859Sml29623 switch (nxge_rx_threshold_lo) { 33343859Sml29623 default: 33353859Sml29623 case NXGE_RX_COPY_NONE: 33363859Sml29623 /* Do not do bcopy at all */ 33373859Sml29623 if (rbrp->rbr_use_bcopy) { 33383859Sml29623 rbrp->rbr_use_bcopy = B_FALSE; 33393859Sml29623 } 33403859Sml29623 rbrp->rbr_threshold_lo = rbrp->rbb_max; 33413859Sml29623 break; 33423859Sml29623 33433859Sml29623 case NXGE_RX_COPY_1: 33443859Sml29623 case NXGE_RX_COPY_2: 33453859Sml29623 case NXGE_RX_COPY_3: 33463859Sml29623 case NXGE_RX_COPY_4: 33473859Sml29623 case NXGE_RX_COPY_5: 33483859Sml29623 case NXGE_RX_COPY_6: 33493859Sml29623 case NXGE_RX_COPY_7: 33503859Sml29623 rbrp->rbr_threshold_lo = 33516929Smisaki rbrp->rbb_max * 33526929Smisaki (nxge_rx_threshold_lo)/NXGE_RX_BCOPY_SCALE; 33533859Sml29623 break; 33543859Sml29623 33553859Sml29623 case NXGE_RX_COPY_ALL: 33563859Sml29623 rbrp->rbr_threshold_lo = 0; 33573859Sml29623 break; 33583859Sml29623 } 33593859Sml29623 33603859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 33616929Smisaki "nxge_map_rxdma_channel_cfg_ring: channel %d " 33626929Smisaki "rbb_max %d " 33636929Smisaki "rbrp->rbr_bufsize_type %d " 33646929Smisaki "rbb_threshold_hi %d " 33656929Smisaki "rbb_threshold_lo %d", 33666929Smisaki dma_channel, 33676929Smisaki rbrp->rbb_max, 33686929Smisaki rbrp->rbr_bufsize_type, 33696929Smisaki rbrp->rbr_threshold_hi, 33706929Smisaki rbrp->rbr_threshold_lo)); 33713859Sml29623 33723859Sml29623 rbrp->page_valid.value = 0; 33733859Sml29623 rbrp->page_mask_1.value = rbrp->page_mask_2.value = 0; 33743859Sml29623 rbrp->page_value_1.value = rbrp->page_value_2.value = 0; 33753859Sml29623 rbrp->page_reloc_1.value = rbrp->page_reloc_2.value = 0; 33763859Sml29623 rbrp->page_hdl.value = 0; 33773859Sml29623 33783859Sml29623 rbrp->page_valid.bits.ldw.page0 = 1; 33793859Sml29623 rbrp->page_valid.bits.ldw.page1 = 1; 33803859Sml29623 33813859Sml29623 /* Map in the receive completion ring */ 33823859Sml29623 rcrp = (p_rx_rcr_ring_t) 33836929Smisaki KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP); 33843859Sml29623 rcrp->rdc = dma_channel; 33853859Sml29623 33863859Sml29623 nxge_port_rcr_size = nxgep->nxge_port_rcr_size; 33873859Sml29623 rcrp->comp_size = nxge_port_rcr_size; 33883859Sml29623 rcrp->comp_wrap_mask = nxge_port_rcr_size - 1; 33893859Sml29623 33903859Sml29623 rcrp->max_receive_pkts = nxge_max_rx_pkts; 33913859Sml29623 33923859Sml29623 dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc; 33933859Sml29623 nxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size, 33946929Smisaki sizeof (rcr_entry_t)); 33953859Sml29623 rcrp->comp_rd_index = 0; 33963859Sml29623 rcrp->comp_wt_index = 0; 33973859Sml29623 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 33986929Smisaki (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 33995125Sjoycey #if defined(__i386) 34006929Smisaki rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 34016929Smisaki (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 34025125Sjoycey #else 34036929Smisaki rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 34046929Smisaki (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 34055125Sjoycey #endif 34063859Sml29623 34073859Sml29623 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 34086929Smisaki (nxge_port_rcr_size - 1); 34093859Sml29623 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 34106929Smisaki (nxge_port_rcr_size - 1); 34113859Sml29623 34123859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 34136929Smisaki "==> nxge_map_rxdma_channel_cfg_ring: " 34146929Smisaki "channel %d " 34156929Smisaki "rbr_vaddrp $%p " 34166929Smisaki "rcr_desc_rd_head_p $%p " 34176929Smisaki "rcr_desc_rd_head_pp $%p " 34186929Smisaki "rcr_desc_rd_last_p $%p " 34196929Smisaki "rcr_desc_rd_last_pp $%p ", 34206929Smisaki dma_channel, 34216929Smisaki rbr_vaddrp, 34226929Smisaki rcrp->rcr_desc_rd_head_p, 34236929Smisaki rcrp->rcr_desc_rd_head_pp, 34246929Smisaki rcrp->rcr_desc_last_p, 34256929Smisaki rcrp->rcr_desc_last_pp)); 34263859Sml29623 34273859Sml29623 /* 34283859Sml29623 * Zero out buffer block ring descriptors. 34293859Sml29623 */ 34303859Sml29623 bzero((caddr_t)dmap->kaddrp, dmap->alength); 34313859Sml29623 rcrp->intr_timeout = nxgep->intr_timeout; 34323859Sml29623 rcrp->intr_threshold = nxgep->intr_threshold; 34333859Sml29623 rcrp->full_hdr_flag = B_FALSE; 34343859Sml29623 rcrp->sw_priv_hdr_len = 0; 34353859Sml29623 34363859Sml29623 cfga_p = &(rcrp->rcr_cfga); 34373859Sml29623 cfgb_p = &(rcrp->rcr_cfgb); 34383859Sml29623 cfga_p->value = 0; 34393859Sml29623 cfgb_p->value = 0; 34403859Sml29623 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress; 34413859Sml29623 cfga_p->value = (rcrp->rcr_addr & 34426929Smisaki (RCRCFIG_A_STADDR_MASK | 34436929Smisaki RCRCFIG_A_STADDR_BASE_MASK)); 34443859Sml29623 34453859Sml29623 rcfga_p->value |= ((uint64_t)rcrp->comp_size << 34466929Smisaki RCRCFIG_A_LEN_SHIF); 34473859Sml29623 34483859Sml29623 /* 34493859Sml29623 * Timeout should be set based on the system clock divider. 34503859Sml29623 * The following timeout value of 1 assumes that the 34513859Sml29623 * granularity (1000) is 3 microseconds running at 300MHz. 34523859Sml29623 */ 34533859Sml29623 cfgb_p->bits.ldw.pthres = rcrp->intr_threshold; 34543859Sml29623 cfgb_p->bits.ldw.timeout = rcrp->intr_timeout; 34553859Sml29623 cfgb_p->bits.ldw.entout = 1; 34563859Sml29623 34573859Sml29623 /* Map in the mailbox */ 34583859Sml29623 mboxp = (p_rx_mbox_t) 34596929Smisaki KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP); 34603859Sml29623 dmap = (p_nxge_dma_common_t)&mboxp->rx_mbox; 34613859Sml29623 nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t)); 34623859Sml29623 cfig1_p = (p_rxdma_cfig1_t)&mboxp->rx_cfg1; 34633859Sml29623 cfig2_p = (p_rxdma_cfig2_t)&mboxp->rx_cfg2; 34643859Sml29623 cfig1_p->value = cfig2_p->value = 0; 34653859Sml29623 34663859Sml29623 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress; 34673859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 34686929Smisaki "==> nxge_map_rxdma_channel_cfg_ring: " 34696929Smisaki "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx", 34706929Smisaki dma_channel, cfig1_p->value, cfig2_p->value, 34716929Smisaki mboxp->mbox_addr)); 34723859Sml29623 34733859Sml29623 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress >> 32 34746929Smisaki & 0xfff); 34753859Sml29623 cfig1_p->bits.ldw.mbaddr_h = dmaaddrp; 34763859Sml29623 34773859Sml29623 34783859Sml29623 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff); 34793859Sml29623 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 34806929Smisaki RXDMA_CFIG2_MBADDR_L_MASK); 34813859Sml29623 34823859Sml29623 cfig2_p->bits.ldw.mbaddr = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT); 34833859Sml29623 34843859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 34856929Smisaki "==> nxge_map_rxdma_channel_cfg_ring: " 34866929Smisaki "channel %d damaddrp $%p " 34876929Smisaki "cfg1 0x%016llx cfig2 0x%016llx", 34886929Smisaki dma_channel, dmaaddrp, 34896929Smisaki cfig1_p->value, cfig2_p->value)); 34903859Sml29623 34913859Sml29623 cfig2_p->bits.ldw.full_hdr = rcrp->full_hdr_flag; 34923859Sml29623 cfig2_p->bits.ldw.offset = rcrp->sw_priv_hdr_len; 34933859Sml29623 34943859Sml29623 rbrp->rx_rcr_p = rcrp; 34953859Sml29623 rcrp->rx_rbr_p = rbrp; 34963859Sml29623 *rcr_p = rcrp; 34973859Sml29623 *rx_mbox_p = mboxp; 34983859Sml29623 34993859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 35006929Smisaki "<== nxge_map_rxdma_channel_cfg_ring status 0x%08x", status)); 35013859Sml29623 35023859Sml29623 return (status); 35033859Sml29623 } 35043859Sml29623 35053859Sml29623 /*ARGSUSED*/ 35063859Sml29623 static void 35073859Sml29623 nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t nxgep, 35083859Sml29623 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p) 35093859Sml29623 { 35103859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 35116929Smisaki "==> nxge_unmap_rxdma_channel_cfg_ring: channel %d", 35126929Smisaki rcr_p->rdc)); 35133859Sml29623 35143859Sml29623 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t)); 35153859Sml29623 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t)); 35163859Sml29623 35173859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 35186929Smisaki "<== nxge_unmap_rxdma_channel_cfg_ring")); 35193859Sml29623 } 35203859Sml29623 35213859Sml29623 static nxge_status_t 35223859Sml29623 nxge_map_rxdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel, 35233859Sml29623 p_nxge_dma_common_t *dma_buf_p, 35243859Sml29623 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks) 35253859Sml29623 { 35263859Sml29623 p_rx_rbr_ring_t rbrp; 35273859Sml29623 p_nxge_dma_common_t dma_bufp, tmp_bufp; 35283859Sml29623 p_rx_msg_t *rx_msg_ring; 35293859Sml29623 p_rx_msg_t rx_msg_p; 35303859Sml29623 p_mblk_t mblk_p; 35313859Sml29623 35323859Sml29623 rxring_info_t *ring_info; 35333859Sml29623 nxge_status_t status = NXGE_OK; 35343859Sml29623 int i, j, index; 35353859Sml29623 uint32_t size, bsize, nblocks, nmsgs; 35363859Sml29623 35373859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 35386929Smisaki "==> nxge_map_rxdma_channel_buf_ring: channel %d", 35396929Smisaki channel)); 35403859Sml29623 35413859Sml29623 dma_bufp = tmp_bufp = *dma_buf_p; 35423859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 35436929Smisaki " nxge_map_rxdma_channel_buf_ring: channel %d to map %d " 35446929Smisaki "chunks bufp 0x%016llx", 35456929Smisaki channel, num_chunks, dma_bufp)); 35463859Sml29623 35473859Sml29623 nmsgs = 0; 35483859Sml29623 for (i = 0; i < num_chunks; i++, tmp_bufp++) { 35493859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 35506929Smisaki "==> nxge_map_rxdma_channel_buf_ring: channel %d " 35516929Smisaki "bufp 0x%016llx nblocks %d nmsgs %d", 35526929Smisaki channel, tmp_bufp, tmp_bufp->nblocks, nmsgs)); 35533859Sml29623 nmsgs += tmp_bufp->nblocks; 35543859Sml29623 } 35553859Sml29623 if (!nmsgs) { 35564185Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 35576929Smisaki "<== nxge_map_rxdma_channel_buf_ring: channel %d " 35586929Smisaki "no msg blocks", 35596929Smisaki channel)); 35603859Sml29623 status = NXGE_ERROR; 35613859Sml29623 goto nxge_map_rxdma_channel_buf_ring_exit; 35623859Sml29623 } 35633859Sml29623 35645170Stm144005 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (*rbrp), KM_SLEEP); 35653859Sml29623 35663859Sml29623 size = nmsgs * sizeof (p_rx_msg_t); 35673859Sml29623 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP); 35683859Sml29623 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t), 35696929Smisaki KM_SLEEP); 35703859Sml29623 35713859Sml29623 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER, 35726929Smisaki (void *)nxgep->interrupt_cookie); 35733859Sml29623 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER, 35746929Smisaki (void *)nxgep->interrupt_cookie); 35753859Sml29623 rbrp->rdc = channel; 35763859Sml29623 rbrp->num_blocks = num_chunks; 35773859Sml29623 rbrp->tnblocks = nmsgs; 35783859Sml29623 rbrp->rbb_max = nmsgs; 35793859Sml29623 rbrp->rbr_max_size = nmsgs; 35803859Sml29623 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1); 35813859Sml29623 35823859Sml29623 /* 35833859Sml29623 * Buffer sizes suggested by NIU architect. 35843859Sml29623 * 256, 512 and 2K. 35853859Sml29623 */ 35863859Sml29623 35873859Sml29623 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B; 35883859Sml29623 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES; 35893859Sml29623 rbrp->npi_pkt_buf_size0 = SIZE_256B; 35903859Sml29623 35913859Sml29623 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K; 35923859Sml29623 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES; 35933859Sml29623 rbrp->npi_pkt_buf_size1 = SIZE_1KB; 35943859Sml29623 35953859Sml29623 rbrp->block_size = nxgep->rx_default_block_size; 35963859Sml29623 35973859Sml29623 if (!nxge_jumbo_enable && !nxgep->param_arr[param_accept_jumbo].value) { 35983859Sml29623 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K; 35993859Sml29623 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES; 36003859Sml29623 rbrp->npi_pkt_buf_size2 = SIZE_2KB; 36013859Sml29623 } else { 36023859Sml29623 if (rbrp->block_size >= 0x2000) { 36033859Sml29623 rbrp->pkt_buf_size2 = RBR_BUFSZ2_8K; 36043859Sml29623 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_8K_BYTES; 36053859Sml29623 rbrp->npi_pkt_buf_size2 = SIZE_8KB; 36063859Sml29623 } else { 36073859Sml29623 rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K; 36083859Sml29623 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES; 36093859Sml29623 rbrp->npi_pkt_buf_size2 = SIZE_4KB; 36103859Sml29623 } 36113859Sml29623 } 36123859Sml29623 36133859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 36146929Smisaki "==> nxge_map_rxdma_channel_buf_ring: channel %d " 36156929Smisaki "actual rbr max %d rbb_max %d nmsgs %d " 36166929Smisaki "rbrp->block_size %d default_block_size %d " 36176929Smisaki "(config nxge_rbr_size %d nxge_rbr_spare_size %d)", 36186929Smisaki channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs, 36196929Smisaki rbrp->block_size, nxgep->rx_default_block_size, 36206929Smisaki nxge_rbr_size, nxge_rbr_spare_size)); 36213859Sml29623 36223859Sml29623 /* Map in buffers from the buffer pool. */ 36233859Sml29623 index = 0; 36243859Sml29623 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) { 36253859Sml29623 bsize = dma_bufp->block_size; 36263859Sml29623 nblocks = dma_bufp->nblocks; 36275125Sjoycey #if defined(__i386) 36285125Sjoycey ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp; 36295125Sjoycey #else 36303859Sml29623 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp; 36315125Sjoycey #endif 36323859Sml29623 ring_info->buffer[i].buf_index = i; 36333859Sml29623 ring_info->buffer[i].buf_size = dma_bufp->alength; 36343859Sml29623 ring_info->buffer[i].start_index = index; 36355125Sjoycey #if defined(__i386) 36365125Sjoycey ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp; 36375125Sjoycey #else 36383859Sml29623 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp; 36395125Sjoycey #endif 36403859Sml29623 36413859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 36426929Smisaki " nxge_map_rxdma_channel_buf_ring: map channel %d " 36436929Smisaki "chunk %d" 36446929Smisaki " nblocks %d chunk_size %x block_size 0x%x " 36456929Smisaki "dma_bufp $%p", channel, i, 36466929Smisaki dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize, 36476929Smisaki dma_bufp)); 36483859Sml29623 36493859Sml29623 for (j = 0; j < nblocks; j++) { 36503859Sml29623 if ((rx_msg_p = nxge_allocb(bsize, BPRI_LO, 36516929Smisaki dma_bufp)) == NULL) { 36524185Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 36536929Smisaki "allocb failed (index %d i %d j %d)", 36546929Smisaki index, i, j)); 36554185Sspeer goto nxge_map_rxdma_channel_buf_ring_fail1; 36563859Sml29623 } 36573859Sml29623 rx_msg_ring[index] = rx_msg_p; 36583859Sml29623 rx_msg_p->block_index = index; 36593859Sml29623 rx_msg_p->shifted_addr = (uint32_t) 36606929Smisaki ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >> 36616929Smisaki RBR_BKADDR_SHIFT)); 36623859Sml29623 36633859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 36646929Smisaki "index %d j %d rx_msg_p $%p mblk %p", 36656929Smisaki index, j, rx_msg_p, rx_msg_p->rx_mblk_p)); 36663859Sml29623 36673859Sml29623 mblk_p = rx_msg_p->rx_mblk_p; 36683859Sml29623 mblk_p->b_wptr = mblk_p->b_rptr + bsize; 36695170Stm144005 36705170Stm144005 rbrp->rbr_ref_cnt++; 36713859Sml29623 index++; 36723859Sml29623 rx_msg_p->buf_dma.dma_channel = channel; 36733859Sml29623 } 36746495Sspeer 36756495Sspeer rbrp->rbr_alloc_type = DDI_MEM_ALLOC; 36766495Sspeer if (dma_bufp->contig_alloc_type) { 36776495Sspeer rbrp->rbr_alloc_type = CONTIG_MEM_ALLOC; 36786495Sspeer } 36796495Sspeer 36806495Sspeer if (dma_bufp->kmem_alloc_type) { 36816495Sspeer rbrp->rbr_alloc_type = KMEM_ALLOC; 36826495Sspeer } 36836495Sspeer 36846495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 36856495Sspeer " nxge_map_rxdma_channel_buf_ring: map channel %d " 36866495Sspeer "chunk %d" 36876495Sspeer " nblocks %d chunk_size %x block_size 0x%x " 36886495Sspeer "dma_bufp $%p", 36896495Sspeer channel, i, 36906495Sspeer dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize, 36916495Sspeer dma_bufp)); 36923859Sml29623 } 36933859Sml29623 if (i < rbrp->num_blocks) { 36943859Sml29623 goto nxge_map_rxdma_channel_buf_ring_fail1; 36953859Sml29623 } 36963859Sml29623 36973859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 36986929Smisaki "nxge_map_rxdma_channel_buf_ring: done buf init " 36996929Smisaki "channel %d msg block entries %d", 37006929Smisaki channel, index)); 37013859Sml29623 ring_info->block_size_mask = bsize - 1; 37023859Sml29623 rbrp->rx_msg_ring = rx_msg_ring; 37033859Sml29623 rbrp->dma_bufp = dma_buf_p; 37043859Sml29623 rbrp->ring_info = ring_info; 37053859Sml29623 37063859Sml29623 status = nxge_rxbuf_index_info_init(nxgep, rbrp); 37073859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37086929Smisaki " nxge_map_rxdma_channel_buf_ring: " 37096929Smisaki "channel %d done buf info init", channel)); 37103859Sml29623 37115170Stm144005 /* 37125170Stm144005 * Finally, permit nxge_freeb() to call nxge_post_page(). 37135170Stm144005 */ 37145170Stm144005 rbrp->rbr_state = RBR_POSTING; 37155170Stm144005 37163859Sml29623 *rbr_p = rbrp; 37173859Sml29623 goto nxge_map_rxdma_channel_buf_ring_exit; 37183859Sml29623 37193859Sml29623 nxge_map_rxdma_channel_buf_ring_fail1: 37203859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37216929Smisaki " nxge_map_rxdma_channel_buf_ring: failed channel (0x%x)", 37226929Smisaki channel, status)); 37233859Sml29623 37243859Sml29623 index--; 37253859Sml29623 for (; index >= 0; index--) { 37263859Sml29623 rx_msg_p = rx_msg_ring[index]; 37273859Sml29623 if (rx_msg_p != NULL) { 37283859Sml29623 freeb(rx_msg_p->rx_mblk_p); 37293859Sml29623 rx_msg_ring[index] = NULL; 37303859Sml29623 } 37313859Sml29623 } 37323859Sml29623 nxge_map_rxdma_channel_buf_ring_fail: 37333859Sml29623 MUTEX_DESTROY(&rbrp->post_lock); 37343859Sml29623 MUTEX_DESTROY(&rbrp->lock); 37353859Sml29623 KMEM_FREE(ring_info, sizeof (rxring_info_t)); 37363859Sml29623 KMEM_FREE(rx_msg_ring, size); 37373859Sml29623 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t)); 37383859Sml29623 37394185Sspeer status = NXGE_ERROR; 37404185Sspeer 37413859Sml29623 nxge_map_rxdma_channel_buf_ring_exit: 37423859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37436929Smisaki "<== nxge_map_rxdma_channel_buf_ring status 0x%08x", status)); 37443859Sml29623 37453859Sml29623 return (status); 37463859Sml29623 } 37473859Sml29623 37483859Sml29623 /*ARGSUSED*/ 37493859Sml29623 static void 37503859Sml29623 nxge_unmap_rxdma_channel_buf_ring(p_nxge_t nxgep, 37513859Sml29623 p_rx_rbr_ring_t rbr_p) 37523859Sml29623 { 37533859Sml29623 p_rx_msg_t *rx_msg_ring; 37543859Sml29623 p_rx_msg_t rx_msg_p; 37553859Sml29623 rxring_info_t *ring_info; 37563859Sml29623 int i; 37573859Sml29623 uint32_t size; 37583859Sml29623 #ifdef NXGE_DEBUG 37593859Sml29623 int num_chunks; 37603859Sml29623 #endif 37613859Sml29623 37623859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37636929Smisaki "==> nxge_unmap_rxdma_channel_buf_ring")); 37643859Sml29623 if (rbr_p == NULL) { 37653859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 37666929Smisaki "<== nxge_unmap_rxdma_channel_buf_ring: NULL rbrp")); 37673859Sml29623 return; 37683859Sml29623 } 37693859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37706929Smisaki "==> nxge_unmap_rxdma_channel_buf_ring: channel %d", 37716929Smisaki rbr_p->rdc)); 37723859Sml29623 37733859Sml29623 rx_msg_ring = rbr_p->rx_msg_ring; 37743859Sml29623 ring_info = rbr_p->ring_info; 37753859Sml29623 37763859Sml29623 if (rx_msg_ring == NULL || ring_info == NULL) { 37776929Smisaki NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37786929Smisaki "<== nxge_unmap_rxdma_channel_buf_ring: " 37796929Smisaki "rx_msg_ring $%p ring_info $%p", 37806929Smisaki rx_msg_p, ring_info)); 37813859Sml29623 return; 37823859Sml29623 } 37833859Sml29623 37843859Sml29623 #ifdef NXGE_DEBUG 37853859Sml29623 num_chunks = rbr_p->num_blocks; 37863859Sml29623 #endif 37873859Sml29623 size = rbr_p->tnblocks * sizeof (p_rx_msg_t); 37883859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37896929Smisaki " nxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d " 37906929Smisaki "tnblocks %d (max %d) size ptrs %d ", 37916929Smisaki rbr_p->rdc, num_chunks, 37926929Smisaki rbr_p->tnblocks, rbr_p->rbr_max_size, size)); 37933859Sml29623 37943859Sml29623 for (i = 0; i < rbr_p->tnblocks; i++) { 37953859Sml29623 rx_msg_p = rx_msg_ring[i]; 37963859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 37976929Smisaki " nxge_unmap_rxdma_channel_buf_ring: " 37986929Smisaki "rx_msg_p $%p", 37996929Smisaki rx_msg_p)); 38003859Sml29623 if (rx_msg_p != NULL) { 38013859Sml29623 freeb(rx_msg_p->rx_mblk_p); 38023859Sml29623 rx_msg_ring[i] = NULL; 38033859Sml29623 } 38043859Sml29623 } 38053859Sml29623 38065170Stm144005 /* 38075170Stm144005 * We no longer may use the mutex <post_lock>. By setting 38085170Stm144005 * <rbr_state> to anything but POSTING, we prevent 38095170Stm144005 * nxge_post_page() from accessing a dead mutex. 38105170Stm144005 */ 38115170Stm144005 rbr_p->rbr_state = RBR_UNMAPPING; 38123859Sml29623 MUTEX_DESTROY(&rbr_p->post_lock); 38135170Stm144005 38143859Sml29623 MUTEX_DESTROY(&rbr_p->lock); 38155170Stm144005 38165170Stm144005 if (rbr_p->rbr_ref_cnt == 0) { 38176495Sspeer /* 38186495Sspeer * This is the normal state of affairs. 38196495Sspeer * Need to free the following buffers: 38206495Sspeer * - data buffers 38216495Sspeer * - rx_msg ring 38226495Sspeer * - ring_info 38236495Sspeer * - rbr ring 38246495Sspeer */ 38256495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, 38266495Sspeer "unmap_rxdma_buf_ring: No outstanding - freeing ")); 38276495Sspeer nxge_rxdma_databuf_free(rbr_p); 38286495Sspeer KMEM_FREE(ring_info, sizeof (rxring_info_t)); 38296495Sspeer KMEM_FREE(rx_msg_ring, size); 38305170Stm144005 KMEM_FREE(rbr_p, sizeof (*rbr_p)); 38315170Stm144005 } else { 38325170Stm144005 /* 38335170Stm144005 * Some of our buffers are still being used. 38345170Stm144005 * Therefore, tell nxge_freeb() this ring is 38355170Stm144005 * unmapped, so it may free <rbr_p> for us. 38365170Stm144005 */ 38375170Stm144005 rbr_p->rbr_state = RBR_UNMAPPED; 38385170Stm144005 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 38395170Stm144005 "unmap_rxdma_buf_ring: %d %s outstanding.", 38405170Stm144005 rbr_p->rbr_ref_cnt, 38415170Stm144005 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs")); 38425170Stm144005 } 38433859Sml29623 38443859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 38456929Smisaki "<== nxge_unmap_rxdma_channel_buf_ring")); 38463859Sml29623 } 38473859Sml29623 38486495Sspeer /* 38496495Sspeer * nxge_rxdma_hw_start_common 38506495Sspeer * 38516495Sspeer * Arguments: 38526495Sspeer * nxgep 38536495Sspeer * 38546495Sspeer * Notes: 38556495Sspeer * 38566495Sspeer * NPI/NXGE function calls: 38576495Sspeer * nxge_init_fzc_rx_common(); 38586495Sspeer * nxge_init_fzc_rxdma_port(); 38596495Sspeer * 38606495Sspeer * Registers accessed: 38616495Sspeer * 38626495Sspeer * Context: 38636495Sspeer * Service domain 38646495Sspeer */ 38653859Sml29623 static nxge_status_t 38663859Sml29623 nxge_rxdma_hw_start_common(p_nxge_t nxgep) 38673859Sml29623 { 38683859Sml29623 nxge_status_t status = NXGE_OK; 38693859Sml29623 38703859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common")); 38713859Sml29623 38723859Sml29623 /* 38733859Sml29623 * Load the sharable parameters by writing to the 38743859Sml29623 * function zero control registers. These FZC registers 38753859Sml29623 * should be initialized only once for the entire chip. 38763859Sml29623 */ 38773859Sml29623 (void) nxge_init_fzc_rx_common(nxgep); 38783859Sml29623 38793859Sml29623 /* 38803859Sml29623 * Initialize the RXDMA port specific FZC control configurations. 38813859Sml29623 * These FZC registers are pertaining to each port. 38823859Sml29623 */ 38833859Sml29623 (void) nxge_init_fzc_rxdma_port(nxgep); 38843859Sml29623 38853859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common")); 38863859Sml29623 38873859Sml29623 return (status); 38883859Sml29623 } 38893859Sml29623 38903859Sml29623 static nxge_status_t 38916495Sspeer nxge_rxdma_hw_start(p_nxge_t nxgep, int channel) 38923859Sml29623 { 38933859Sml29623 int i, ndmas; 38943859Sml29623 p_rx_rbr_rings_t rx_rbr_rings; 38953859Sml29623 p_rx_rbr_ring_t *rbr_rings; 38963859Sml29623 p_rx_rcr_rings_t rx_rcr_rings; 38973859Sml29623 p_rx_rcr_ring_t *rcr_rings; 38983859Sml29623 p_rx_mbox_areas_t rx_mbox_areas_p; 38993859Sml29623 p_rx_mbox_t *rx_mbox_p; 39003859Sml29623 nxge_status_t status = NXGE_OK; 39013859Sml29623 39023859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start")); 39033859Sml29623 39043859Sml29623 rx_rbr_rings = nxgep->rx_rbr_rings; 39053859Sml29623 rx_rcr_rings = nxgep->rx_rcr_rings; 39063859Sml29623 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 39073859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 39086929Smisaki "<== nxge_rxdma_hw_start: NULL ring pointers")); 39093859Sml29623 return (NXGE_ERROR); 39103859Sml29623 } 39113859Sml29623 ndmas = rx_rbr_rings->ndmas; 39123859Sml29623 if (ndmas == 0) { 39133859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 39146929Smisaki "<== nxge_rxdma_hw_start: no dma channel allocated")); 39153859Sml29623 return (NXGE_ERROR); 39163859Sml29623 } 39173859Sml29623 39183859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 39196929Smisaki "==> nxge_rxdma_hw_start (ndmas %d)", ndmas)); 39203859Sml29623 39213859Sml29623 rbr_rings = rx_rbr_rings->rbr_rings; 39223859Sml29623 rcr_rings = rx_rcr_rings->rcr_rings; 39233859Sml29623 rx_mbox_areas_p = nxgep->rx_mbox_areas_p; 39243859Sml29623 if (rx_mbox_areas_p) { 39253859Sml29623 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas; 39263859Sml29623 } 39273859Sml29623 39286495Sspeer i = channel; 39296495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 39306929Smisaki "==> nxge_rxdma_hw_start (ndmas %d) channel %d", 39316929Smisaki ndmas, channel)); 39326495Sspeer status = nxge_rxdma_start_channel(nxgep, channel, 39336495Sspeer (p_rx_rbr_ring_t)rbr_rings[i], 39346495Sspeer (p_rx_rcr_ring_t)rcr_rings[i], 39356495Sspeer (p_rx_mbox_t)rx_mbox_p[i]); 39366495Sspeer if (status != NXGE_OK) { 39376495Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 39386495Sspeer "==> nxge_rxdma_hw_start: disable " 39396495Sspeer "(status 0x%x channel %d)", status, channel)); 39406495Sspeer return (status); 39413859Sml29623 } 39423859Sml29623 39433859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start: " 39446929Smisaki "rx_rbr_rings 0x%016llx rings 0x%016llx", 39456929Smisaki rx_rbr_rings, rx_rcr_rings)); 39463859Sml29623 39473859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 39486929Smisaki "==> nxge_rxdma_hw_start: (status 0x%x)", status)); 39493859Sml29623 39503859Sml29623 return (status); 39513859Sml29623 } 39523859Sml29623 39533859Sml29623 static void 39546495Sspeer nxge_rxdma_hw_stop(p_nxge_t nxgep, int channel) 39553859Sml29623 { 39563859Sml29623 p_rx_rbr_rings_t rx_rbr_rings; 39573859Sml29623 p_rx_rcr_rings_t rx_rcr_rings; 39583859Sml29623 39593859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop")); 39603859Sml29623 39613859Sml29623 rx_rbr_rings = nxgep->rx_rbr_rings; 39623859Sml29623 rx_rcr_rings = nxgep->rx_rcr_rings; 39633859Sml29623 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) { 39643859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 39656929Smisaki "<== nxge_rxdma_hw_stop: NULL ring pointers")); 39663859Sml29623 return; 39673859Sml29623 } 39683859Sml29623 39693859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 39706929Smisaki "==> nxge_rxdma_hw_stop(channel %d)", 39716929Smisaki channel)); 39726495Sspeer (void) nxge_rxdma_stop_channel(nxgep, channel); 39733859Sml29623 39743859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop: " 39756929Smisaki "rx_rbr_rings 0x%016llx rings 0x%016llx", 39766929Smisaki rx_rbr_rings, rx_rcr_rings)); 39773859Sml29623 39783859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_hw_stop")); 39793859Sml29623 } 39803859Sml29623 39813859Sml29623 39823859Sml29623 static nxge_status_t 39833859Sml29623 nxge_rxdma_start_channel(p_nxge_t nxgep, uint16_t channel, 39843859Sml29623 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p) 39853859Sml29623 39863859Sml29623 { 39873859Sml29623 npi_handle_t handle; 39883859Sml29623 npi_status_t rs = NPI_SUCCESS; 39893859Sml29623 rx_dma_ctl_stat_t cs; 39903859Sml29623 rx_dma_ent_msk_t ent_mask; 39913859Sml29623 nxge_status_t status = NXGE_OK; 39923859Sml29623 39933859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel")); 39943859Sml29623 39953859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 39963859Sml29623 39973859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "nxge_rxdma_start_channel: " 39983859Sml29623 "npi handle addr $%p acc $%p", 39993859Sml29623 nxgep->npi_handle.regp, nxgep->npi_handle.regh)); 40003859Sml29623 40016495Sspeer /* Reset RXDMA channel, but not if you're a guest. */ 40026495Sspeer if (!isLDOMguest(nxgep)) { 40036495Sspeer rs = npi_rxdma_cfg_rdc_reset(handle, channel); 40046495Sspeer if (rs != NPI_SUCCESS) { 40056495Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 40066495Sspeer "==> nxge_init_fzc_rdc: " 40076495Sspeer "npi_rxdma_cfg_rdc_reset(%d) returned 0x%08x", 40086495Sspeer channel, rs)); 40096495Sspeer return (NXGE_ERROR | rs); 40106495Sspeer } 40116495Sspeer 40126495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 40136495Sspeer "==> nxge_rxdma_start_channel: reset done: channel %d", 40146495Sspeer channel)); 40153859Sml29623 } 40163859Sml29623 40176495Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 40186495Sspeer if (isLDOMguest(nxgep)) 40196495Sspeer (void) nxge_rdc_lp_conf(nxgep, channel); 40206495Sspeer #endif 40213859Sml29623 40223859Sml29623 /* 40233859Sml29623 * Initialize the RXDMA channel specific FZC control 40243859Sml29623 * configurations. These FZC registers are pertaining 40253859Sml29623 * to each RX channel (logical pages). 40263859Sml29623 */ 40276495Sspeer if (!isLDOMguest(nxgep)) { 40286495Sspeer status = nxge_init_fzc_rxdma_channel(nxgep, channel); 40296495Sspeer if (status != NXGE_OK) { 40306495Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 40316495Sspeer "==> nxge_rxdma_start_channel: " 40326495Sspeer "init fzc rxdma failed (0x%08x channel %d)", 40336495Sspeer status, channel)); 40346495Sspeer return (status); 40356495Sspeer } 40366495Sspeer 40376495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 40386495Sspeer "==> nxge_rxdma_start_channel: fzc done")); 40393859Sml29623 } 40403859Sml29623 40413859Sml29623 /* Set up the interrupt event masks. */ 40423859Sml29623 ent_mask.value = 0; 40433859Sml29623 ent_mask.value |= RX_DMA_ENT_MSK_RBREMPTY_MASK; 40443859Sml29623 rs = npi_rxdma_event_mask(handle, OP_SET, channel, 40456495Sspeer &ent_mask); 40463859Sml29623 if (rs != NPI_SUCCESS) { 40473859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 40483859Sml29623 "==> nxge_rxdma_start_channel: " 40496495Sspeer "init rxdma event masks failed " 40506495Sspeer "(0x%08x channel %d)", 40513859Sml29623 status, channel)); 40523859Sml29623 return (NXGE_ERROR | rs); 40533859Sml29623 } 40543859Sml29623 40556495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 40566495Sspeer "==> nxge_rxdma_start_channel: " 40573859Sml29623 "event done: channel %d (mask 0x%016llx)", 40583859Sml29623 channel, ent_mask.value)); 40593859Sml29623 40603859Sml29623 /* Initialize the receive DMA control and status register */ 40613859Sml29623 cs.value = 0; 40623859Sml29623 cs.bits.hdw.mex = 1; 40633859Sml29623 cs.bits.hdw.rcrthres = 1; 40643859Sml29623 cs.bits.hdw.rcrto = 1; 40653859Sml29623 cs.bits.hdw.rbr_empty = 1; 40663859Sml29623 status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs); 40673859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: " 40683859Sml29623 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value)); 40693859Sml29623 if (status != NXGE_OK) { 40703859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 40713859Sml29623 "==> nxge_rxdma_start_channel: " 40723859Sml29623 "init rxdma control register failed (0x%08x channel %d", 40733859Sml29623 status, channel)); 40743859Sml29623 return (status); 40753859Sml29623 } 40763859Sml29623 40773859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: " 40783859Sml29623 "control done - channel %d cs 0x%016llx", channel, cs.value)); 40793859Sml29623 40803859Sml29623 /* 40813859Sml29623 * Load RXDMA descriptors, buffers, mailbox, 40823859Sml29623 * initialise the receive DMA channels and 40833859Sml29623 * enable each DMA channel. 40843859Sml29623 */ 40853859Sml29623 status = nxge_enable_rxdma_channel(nxgep, 40866495Sspeer channel, rbr_p, rcr_p, mbox_p); 40873859Sml29623 40883859Sml29623 if (status != NXGE_OK) { 40893859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 40906495Sspeer " nxge_rxdma_start_channel: " 40916495Sspeer " enable rxdma failed (0x%08x channel %d)", 40926495Sspeer status, channel)); 40936495Sspeer return (status); 40946495Sspeer } 40956495Sspeer 40966495Sspeer NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 40976495Sspeer "==> nxge_rxdma_start_channel: enabled channel %d")); 40986495Sspeer 40996495Sspeer if (isLDOMguest(nxgep)) { 41006495Sspeer /* Add interrupt handler for this channel. */ 41016495Sspeer if (nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel) 41026495Sspeer != NXGE_OK) { 41036495Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 41043859Sml29623 " nxge_rxdma_start_channel: " 41056495Sspeer " nxge_hio_intr_add failed (0x%08x channel %d)", 41066495Sspeer status, channel)); 41076495Sspeer } 41083859Sml29623 } 41093859Sml29623 41103859Sml29623 ent_mask.value = 0; 41113859Sml29623 ent_mask.value |= (RX_DMA_ENT_MSK_WRED_DROP_MASK | 41123859Sml29623 RX_DMA_ENT_MSK_PTDROP_PKT_MASK); 41133859Sml29623 rs = npi_rxdma_event_mask(handle, OP_SET, channel, 41143859Sml29623 &ent_mask); 41153859Sml29623 if (rs != NPI_SUCCESS) { 41163859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, 41173859Sml29623 "==> nxge_rxdma_start_channel: " 41183859Sml29623 "init rxdma event masks failed (0x%08x channel %d)", 41193859Sml29623 status, channel)); 41203859Sml29623 return (NXGE_ERROR | rs); 41213859Sml29623 } 41223859Sml29623 41233859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: " 41243859Sml29623 "control done - channel %d cs 0x%016llx", channel, cs.value)); 41253859Sml29623 41263859Sml29623 NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_start_channel")); 41273859Sml29623 41283859Sml29623 return (NXGE_OK); 41293859Sml29623 } 41303859Sml29623 41313859Sml29623 static nxge_status_t 41323859Sml29623 nxge_rxdma_stop_channel(p_nxge_t nxgep, uint16_t channel) 41333859Sml29623 { 41343859Sml29623 npi_handle_t handle; 41353859Sml29623 npi_status_t rs = NPI_SUCCESS; 41363859Sml29623 rx_dma_ctl_stat_t cs; 41373859Sml29623 rx_dma_ent_msk_t ent_mask; 41383859Sml29623 nxge_status_t status = NXGE_OK; 41393859Sml29623 41403859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel")); 41413859Sml29623 41423859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 41433859Sml29623 41443859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "nxge_rxdma_stop_channel: " 41456929Smisaki "npi handle addr $%p acc $%p", 41466929Smisaki nxgep->npi_handle.regp, nxgep->npi_handle.regh)); 41473859Sml29623 4148*7812SMichael.Speer@Sun.COM if (!isLDOMguest(nxgep)) { 4149*7812SMichael.Speer@Sun.COM /* 4150*7812SMichael.Speer@Sun.COM * Stop RxMAC = A.9.2.6 4151*7812SMichael.Speer@Sun.COM */ 4152*7812SMichael.Speer@Sun.COM if (nxge_rx_mac_disable(nxgep) != NXGE_OK) { 4153*7812SMichael.Speer@Sun.COM NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4154*7812SMichael.Speer@Sun.COM "nxge_rxdma_stop_channel: " 4155*7812SMichael.Speer@Sun.COM "Failed to disable RxMAC")); 4156*7812SMichael.Speer@Sun.COM } 4157*7812SMichael.Speer@Sun.COM 4158*7812SMichael.Speer@Sun.COM /* 4159*7812SMichael.Speer@Sun.COM * Drain IPP Port = A.9.3.6 4160*7812SMichael.Speer@Sun.COM */ 4161*7812SMichael.Speer@Sun.COM (void) nxge_ipp_drain(nxgep); 4162*7812SMichael.Speer@Sun.COM } 4163*7812SMichael.Speer@Sun.COM 41643859Sml29623 /* Reset RXDMA channel */ 41653859Sml29623 rs = npi_rxdma_cfg_rdc_reset(handle, channel); 41663859Sml29623 if (rs != NPI_SUCCESS) { 41673859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 41686929Smisaki " nxge_rxdma_stop_channel: " 41696929Smisaki " reset rxdma failed (0x%08x channel %d)", 41706929Smisaki rs, channel)); 41713859Sml29623 return (NXGE_ERROR | rs); 41723859Sml29623 } 41733859Sml29623 41743859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 41756929Smisaki "==> nxge_rxdma_stop_channel: reset done")); 41763859Sml29623 41773859Sml29623 /* Set up the interrupt event masks. */ 41783859Sml29623 ent_mask.value = RX_DMA_ENT_MSK_ALL; 41793859Sml29623 rs = npi_rxdma_event_mask(handle, OP_SET, channel, 41806929Smisaki &ent_mask); 41813859Sml29623 if (rs != NPI_SUCCESS) { 41823859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 41836929Smisaki "==> nxge_rxdma_stop_channel: " 41846929Smisaki "set rxdma event masks failed (0x%08x channel %d)", 41856929Smisaki rs, channel)); 41863859Sml29623 return (NXGE_ERROR | rs); 41873859Sml29623 } 41883859Sml29623 41893859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 41906929Smisaki "==> nxge_rxdma_stop_channel: event done")); 41913859Sml29623 4192*7812SMichael.Speer@Sun.COM /* 4193*7812SMichael.Speer@Sun.COM * Initialize the receive DMA control and status register 4194*7812SMichael.Speer@Sun.COM */ 41953859Sml29623 cs.value = 0; 4196*7812SMichael.Speer@Sun.COM status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs); 41973859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel: control " 41986929Smisaki " to default (all 0s) 0x%08x", cs.value)); 41993859Sml29623 if (status != NXGE_OK) { 42003859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 42016929Smisaki " nxge_rxdma_stop_channel: init rxdma" 42026929Smisaki " control register failed (0x%08x channel %d", 42036929Smisaki status, channel)); 42043859Sml29623 return (status); 42053859Sml29623 } 42063859Sml29623 42073859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, 42086929Smisaki "==> nxge_rxdma_stop_channel: control done")); 42093859Sml29623 4210*7812SMichael.Speer@Sun.COM /* 4211*7812SMichael.Speer@Sun.COM * Make sure channel is disabled. 4212*7812SMichael.Speer@Sun.COM */ 42133859Sml29623 status = nxge_disable_rxdma_channel(nxgep, channel); 42143859Sml29623 if (status != NXGE_OK) { 42153859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 42166929Smisaki " nxge_rxdma_stop_channel: " 42176929Smisaki " init enable rxdma failed (0x%08x channel %d)", 42186929Smisaki status, channel)); 42193859Sml29623 return (status); 42203859Sml29623 } 42213859Sml29623 4222*7812SMichael.Speer@Sun.COM if (!isLDOMguest(nxgep)) { 4223*7812SMichael.Speer@Sun.COM /* 4224*7812SMichael.Speer@Sun.COM * Enable RxMAC = A.9.2.10 4225*7812SMichael.Speer@Sun.COM */ 4226*7812SMichael.Speer@Sun.COM if (nxge_rx_mac_enable(nxgep) != NXGE_OK) { 4227*7812SMichael.Speer@Sun.COM NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 4228*7812SMichael.Speer@Sun.COM "nxge_rxdma_stop_channel: Rx MAC still disabled")); 4229*7812SMichael.Speer@Sun.COM } 4230*7812SMichael.Speer@Sun.COM } 4231*7812SMichael.Speer@Sun.COM 42323859Sml29623 NXGE_DEBUG_MSG((nxgep, 42336929Smisaki RX_CTL, "==> nxge_rxdma_stop_channel: disable done")); 42343859Sml29623 42353859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_channel")); 42363859Sml29623 42373859Sml29623 return (NXGE_OK); 42383859Sml29623 } 42393859Sml29623 42403859Sml29623 nxge_status_t 42413859Sml29623 nxge_rxdma_handle_sys_errors(p_nxge_t nxgep) 42423859Sml29623 { 42433859Sml29623 npi_handle_t handle; 42443859Sml29623 p_nxge_rdc_sys_stats_t statsp; 42453859Sml29623 rx_ctl_dat_fifo_stat_t stat; 42463859Sml29623 uint32_t zcp_err_status; 42473859Sml29623 uint32_t ipp_err_status; 42483859Sml29623 nxge_status_t status = NXGE_OK; 42493859Sml29623 npi_status_t rs = NPI_SUCCESS; 42503859Sml29623 boolean_t my_err = B_FALSE; 42513859Sml29623 42523859Sml29623 handle = nxgep->npi_handle; 42533859Sml29623 statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats; 42543859Sml29623 42553859Sml29623 rs = npi_rxdma_rxctl_fifo_error_intr_get(handle, &stat); 42563859Sml29623 42573859Sml29623 if (rs != NPI_SUCCESS) 42583859Sml29623 return (NXGE_ERROR | rs); 42593859Sml29623 42603859Sml29623 if (stat.bits.ldw.id_mismatch) { 42613859Sml29623 statsp->id_mismatch++; 42623859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL, 42636929Smisaki NXGE_FM_EREPORT_RDMC_ID_MISMATCH); 42643859Sml29623 /* Global fatal error encountered */ 42653859Sml29623 } 42663859Sml29623 42673859Sml29623 if ((stat.bits.ldw.zcp_eop_err) || (stat.bits.ldw.ipp_eop_err)) { 42683859Sml29623 switch (nxgep->mac.portnum) { 42693859Sml29623 case 0: 42703859Sml29623 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT0) || 42716929Smisaki (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT0)) { 42723859Sml29623 my_err = B_TRUE; 42733859Sml29623 zcp_err_status = stat.bits.ldw.zcp_eop_err; 42743859Sml29623 ipp_err_status = stat.bits.ldw.ipp_eop_err; 42753859Sml29623 } 42763859Sml29623 break; 42773859Sml29623 case 1: 42783859Sml29623 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT1) || 42796929Smisaki (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT1)) { 42803859Sml29623 my_err = B_TRUE; 42813859Sml29623 zcp_err_status = stat.bits.ldw.zcp_eop_err; 42823859Sml29623 ipp_err_status = stat.bits.ldw.ipp_eop_err; 42833859Sml29623 } 42843859Sml29623 break; 42853859Sml29623 case 2: 42863859Sml29623 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT2) || 42876929Smisaki (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT2)) { 42883859Sml29623 my_err = B_TRUE; 42893859Sml29623 zcp_err_status = stat.bits.ldw.zcp_eop_err; 42903859Sml29623 ipp_err_status = stat.bits.ldw.ipp_eop_err; 42913859Sml29623 } 42923859Sml29623 break; 42933859Sml29623 case 3: 42943859Sml29623 if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT3) || 42956929Smisaki (stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT3)) { 42963859Sml29623 my_err = B_TRUE; 42973859Sml29623 zcp_err_status = stat.bits.ldw.zcp_eop_err; 42983859Sml29623 ipp_err_status = stat.bits.ldw.ipp_eop_err; 42993859Sml29623 } 43003859Sml29623 break; 43013859Sml29623 default: 43023859Sml29623 return (NXGE_ERROR); 43033859Sml29623 } 43043859Sml29623 } 43053859Sml29623 43063859Sml29623 if (my_err) { 43073859Sml29623 status = nxge_rxdma_handle_port_errors(nxgep, ipp_err_status, 43086929Smisaki zcp_err_status); 43093859Sml29623 if (status != NXGE_OK) 43103859Sml29623 return (status); 43113859Sml29623 } 43123859Sml29623 43133859Sml29623 return (NXGE_OK); 43143859Sml29623 } 43153859Sml29623 43163859Sml29623 static nxge_status_t 43173859Sml29623 nxge_rxdma_handle_port_errors(p_nxge_t nxgep, uint32_t ipp_status, 43183859Sml29623 uint32_t zcp_status) 43193859Sml29623 { 43203859Sml29623 boolean_t rxport_fatal = B_FALSE; 43213859Sml29623 p_nxge_rdc_sys_stats_t statsp; 43223859Sml29623 nxge_status_t status = NXGE_OK; 43233859Sml29623 uint8_t portn; 43243859Sml29623 43253859Sml29623 portn = nxgep->mac.portnum; 43263859Sml29623 statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats; 43273859Sml29623 43283859Sml29623 if (ipp_status & (0x1 << portn)) { 43293859Sml29623 statsp->ipp_eop_err++; 43303859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, NULL, 43316929Smisaki NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR); 43323859Sml29623 rxport_fatal = B_TRUE; 43333859Sml29623 } 43343859Sml29623 43353859Sml29623 if (zcp_status & (0x1 << portn)) { 43363859Sml29623 statsp->zcp_eop_err++; 43373859Sml29623 NXGE_FM_REPORT_ERROR(nxgep, portn, NULL, 43386929Smisaki NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR); 43393859Sml29623 rxport_fatal = B_TRUE; 43403859Sml29623 } 43413859Sml29623 43423859Sml29623 if (rxport_fatal) { 43433859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 43446929Smisaki " nxge_rxdma_handle_port_error: " 43456929Smisaki " fatal error on Port #%d\n", 43466929Smisaki portn)); 43473859Sml29623 status = nxge_rx_port_fatal_err_recover(nxgep); 43483859Sml29623 if (status == NXGE_OK) { 43493859Sml29623 FM_SERVICE_RESTORED(nxgep); 43503859Sml29623 } 43513859Sml29623 } 43523859Sml29623 43533859Sml29623 return (status); 43543859Sml29623 } 43553859Sml29623 43563859Sml29623 static nxge_status_t 43573859Sml29623 nxge_rxdma_fatal_err_recover(p_nxge_t nxgep, uint16_t channel) 43583859Sml29623 { 43593859Sml29623 npi_handle_t handle; 43603859Sml29623 npi_status_t rs = NPI_SUCCESS; 43613859Sml29623 nxge_status_t status = NXGE_OK; 43623859Sml29623 p_rx_rbr_ring_t rbrp; 43633859Sml29623 p_rx_rcr_ring_t rcrp; 43643859Sml29623 p_rx_mbox_t mboxp; 43653859Sml29623 rx_dma_ent_msk_t ent_mask; 43663859Sml29623 p_nxge_dma_common_t dmap; 43673859Sml29623 int ring_idx; 43683859Sml29623 uint32_t ref_cnt; 43693859Sml29623 p_rx_msg_t rx_msg_p; 43703859Sml29623 int i; 43713859Sml29623 uint32_t nxge_port_rcr_size; 43723859Sml29623 43733859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fatal_err_recover")); 43743859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 43756929Smisaki "Recovering from RxDMAChannel#%d error...", channel)); 43763859Sml29623 43773859Sml29623 /* 43783859Sml29623 * Stop the dma channel waits for the stop done. 43793859Sml29623 * If the stop done bit is not set, then create 43803859Sml29623 * an error. 43813859Sml29623 */ 43823859Sml29623 43833859Sml29623 handle = NXGE_DEV_NPI_HANDLE(nxgep); 43843859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Rx DMA stop...")); 43853859Sml29623 43863859Sml29623 ring_idx = nxge_rxdma_get_ring_index(nxgep, channel); 43873859Sml29623 rbrp = (p_rx_rbr_ring_t)nxgep->rx_rbr_rings->rbr_rings[ring_idx]; 43883859Sml29623 rcrp = (p_rx_rcr_ring_t)nxgep->rx_rcr_rings->rcr_rings[ring_idx]; 43893859Sml29623 43903859Sml29623 MUTEX_ENTER(&rcrp->lock); 43913859Sml29623 MUTEX_ENTER(&rbrp->lock); 43923859Sml29623 MUTEX_ENTER(&rbrp->post_lock); 43933859Sml29623 43943859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA channel...")); 43953859Sml29623 43963859Sml29623 rs = npi_rxdma_cfg_rdc_disable(handle, channel); 43973859Sml29623 if (rs != NPI_SUCCESS) { 43983859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 43996929Smisaki "nxge_disable_rxdma_channel:failed")); 44003859Sml29623 goto fail; 44013859Sml29623 } 44023859Sml29623 44033859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA interrupt...")); 44043859Sml29623 44053859Sml29623 /* Disable interrupt */ 44063859Sml29623 ent_mask.value = RX_DMA_ENT_MSK_ALL; 44073859Sml29623 rs = npi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask); 44083859Sml29623 if (rs != NPI_SUCCESS) { 44093859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 44106929Smisaki "nxge_rxdma_stop_channel: " 44116929Smisaki "set rxdma event masks failed (channel %d)", 44126929Smisaki channel)); 44133859Sml29623 } 44143859Sml29623 44153859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel reset...")); 44163859Sml29623 44173859Sml29623 /* Reset RXDMA channel */ 44183859Sml29623 rs = npi_rxdma_cfg_rdc_reset(handle, channel); 44193859Sml29623 if (rs != NPI_SUCCESS) { 44203859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 44216929Smisaki "nxge_rxdma_fatal_err_recover: " 44226929Smisaki " reset rxdma failed (channel %d)", channel)); 44233859Sml29623 goto fail; 44243859Sml29623 } 44253859Sml29623 44263859Sml29623 nxge_port_rcr_size = nxgep->nxge_port_rcr_size; 44273859Sml29623 44283859Sml29623 mboxp = 44296929Smisaki (p_rx_mbox_t)nxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx]; 44303859Sml29623 44313859Sml29623 rbrp->rbr_wr_index = (rbrp->rbb_max - 1); 44323859Sml29623 rbrp->rbr_rd_index = 0; 44333859Sml29623 44343859Sml29623 rcrp->comp_rd_index = 0; 44353859Sml29623 rcrp->comp_wt_index = 0; 44363859Sml29623 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p = 44376929Smisaki (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc); 44385125Sjoycey #if defined(__i386) 44396929Smisaki rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 44406929Smisaki (p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 44415125Sjoycey #else 44426929Smisaki rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp = 44436929Smisaki (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc); 44445125Sjoycey #endif 44453859Sml29623 44463859Sml29623 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p + 44476929Smisaki (nxge_port_rcr_size - 1); 44483859Sml29623 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp + 44496929Smisaki (nxge_port_rcr_size - 1); 44503859Sml29623 44513859Sml29623 dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc; 44523859Sml29623 bzero((caddr_t)dmap->kaddrp, dmap->alength); 44533859Sml29623 44543859Sml29623 cmn_err(CE_NOTE, "!rbr entries = %d\n", rbrp->rbr_max_size); 44553859Sml29623 44563859Sml29623 for (i = 0; i < rbrp->rbr_max_size; i++) { 44573859Sml29623 rx_msg_p = rbrp->rx_msg_ring[i]; 44583859Sml29623 ref_cnt = rx_msg_p->ref_cnt; 44593859Sml29623 if (ref_cnt != 1) { 44603859Sml29623 if (rx_msg_p->cur_usage_cnt != 44616929Smisaki rx_msg_p->max_usage_cnt) { 44623859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 44636929Smisaki "buf[%d]: cur_usage_cnt = %d " 44646929Smisaki "max_usage_cnt = %d\n", i, 44656929Smisaki rx_msg_p->cur_usage_cnt, 44666929Smisaki rx_msg_p->max_usage_cnt)); 44673859Sml29623 } else { 44683859Sml29623 /* Buffer can be re-posted */ 44693859Sml29623 rx_msg_p->free = B_TRUE; 44703859Sml29623 rx_msg_p->cur_usage_cnt = 0; 44713859Sml29623 rx_msg_p->max_usage_cnt = 0xbaddcafe; 44723859Sml29623 rx_msg_p->pkt_buf_size = 0; 44733859Sml29623 } 44743859Sml29623 } 44753859Sml29623 } 44763859Sml29623 44773859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel re-start...")); 44783859Sml29623 44793859Sml29623 status = nxge_rxdma_start_channel(nxgep, channel, rbrp, rcrp, mboxp); 44803859Sml29623 if (status != NXGE_OK) { 44813859Sml29623 goto fail; 44823859Sml29623 } 44833859Sml29623 44843859Sml29623 MUTEX_EXIT(&rbrp->post_lock); 44853859Sml29623 MUTEX_EXIT(&rbrp->lock); 44863859Sml29623 MUTEX_EXIT(&rcrp->lock); 44873859Sml29623 44883859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 44896929Smisaki "Recovery Successful, RxDMAChannel#%d Restored", 44906929Smisaki channel)); 44913859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fatal_err_recover")); 44923859Sml29623 44933859Sml29623 return (NXGE_OK); 44943859Sml29623 fail: 44953859Sml29623 MUTEX_EXIT(&rbrp->post_lock); 44963859Sml29623 MUTEX_EXIT(&rbrp->lock); 44973859Sml29623 MUTEX_EXIT(&rcrp->lock); 44983859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed")); 44993859Sml29623 45003859Sml29623 return (NXGE_ERROR | rs); 45013859Sml29623 } 45023859Sml29623 45033859Sml29623 nxge_status_t 45043859Sml29623 nxge_rx_port_fatal_err_recover(p_nxge_t nxgep) 45053859Sml29623 { 45066495Sspeer nxge_grp_set_t *set = &nxgep->rx_set; 45076495Sspeer nxge_status_t status = NXGE_OK; 45086495Sspeer int rdc; 45093859Sml29623 45103859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_port_fatal_err_recover")); 45113859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45126929Smisaki "Recovering from RxPort error...")); 45136495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disabling RxMAC...\n")); 45146495Sspeer 45153859Sml29623 if (nxge_rx_mac_disable(nxgep) != NXGE_OK) 45163859Sml29623 goto fail; 45173859Sml29623 45183859Sml29623 NXGE_DELAY(1000); 45193859Sml29623 45206495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, "Stopping all RxDMA channels...")); 45216495Sspeer 45226495Sspeer for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) { 45236495Sspeer if ((1 << rdc) & set->owned.map) { 45246495Sspeer if (nxge_rxdma_fatal_err_recover(nxgep, rdc) 45256495Sspeer != NXGE_OK) { 45266495Sspeer NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45276495Sspeer "Could not recover channel %d", rdc)); 45286495Sspeer } 45293859Sml29623 } 45303859Sml29623 } 45313859Sml29623 45326495Sspeer NXGE_DEBUG_MSG((nxgep, RX_CTL, "Resetting IPP...")); 45333859Sml29623 45343859Sml29623 /* Reset IPP */ 45353859Sml29623 if (nxge_ipp_reset(nxgep) != NXGE_OK) { 45363859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45376929Smisaki "nxge_rx_port_fatal_err_recover: " 45386929Smisaki "Failed to reset IPP")); 45393859Sml29623 goto fail; 45403859Sml29623 } 45413859Sml29623 45423859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Reset RxMAC...")); 45433859Sml29623 45443859Sml29623 /* Reset RxMAC */ 45453859Sml29623 if (nxge_rx_mac_reset(nxgep) != NXGE_OK) { 45463859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45476929Smisaki "nxge_rx_port_fatal_err_recover: " 45486929Smisaki "Failed to reset RxMAC")); 45493859Sml29623 goto fail; 45503859Sml29623 } 45513859Sml29623 45523859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize IPP...")); 45533859Sml29623 45543859Sml29623 /* Re-Initialize IPP */ 45553859Sml29623 if (nxge_ipp_init(nxgep) != NXGE_OK) { 45563859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45576929Smisaki "nxge_rx_port_fatal_err_recover: " 45586929Smisaki "Failed to init IPP")); 45593859Sml29623 goto fail; 45603859Sml29623 } 45613859Sml29623 45623859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize RxMAC...")); 45633859Sml29623 45643859Sml29623 /* Re-Initialize RxMAC */ 45653859Sml29623 if ((status = nxge_rx_mac_init(nxgep)) != NXGE_OK) { 45663859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45676929Smisaki "nxge_rx_port_fatal_err_recover: " 45686929Smisaki "Failed to reset RxMAC")); 45693859Sml29623 goto fail; 45703859Sml29623 } 45713859Sml29623 45723859Sml29623 NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-enable RxMAC...")); 45733859Sml29623 45743859Sml29623 /* Re-enable RxMAC */ 45753859Sml29623 if ((status = nxge_rx_mac_enable(nxgep)) != NXGE_OK) { 45763859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45776929Smisaki "nxge_rx_port_fatal_err_recover: " 45786929Smisaki "Failed to enable RxMAC")); 45793859Sml29623 goto fail; 45803859Sml29623 } 45813859Sml29623 45823859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, 45836929Smisaki "Recovery Successful, RxPort Restored")); 45843859Sml29623 45853859Sml29623 return (NXGE_OK); 45863859Sml29623 fail: 45873859Sml29623 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed")); 45883859Sml29623 return (status); 45893859Sml29623 } 45903859Sml29623 45913859Sml29623 void 45923859Sml29623 nxge_rxdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan) 45933859Sml29623 { 45943859Sml29623 rx_dma_ctl_stat_t cs; 45953859Sml29623 rx_ctl_dat_fifo_stat_t cdfs; 45963859Sml29623 45973859Sml29623 switch (err_id) { 45983859Sml29623 case NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR: 45993859Sml29623 case NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR: 46003859Sml29623 case NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR: 46013859Sml29623 case NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR: 46023859Sml29623 case NXGE_FM_EREPORT_RDMC_RBR_TMOUT: 46033859Sml29623 case NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR: 46043859Sml29623 case NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS: 46053859Sml29623 case NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR: 46063859Sml29623 case NXGE_FM_EREPORT_RDMC_RCRINCON: 46073859Sml29623 case NXGE_FM_EREPORT_RDMC_RCRFULL: 46083859Sml29623 case NXGE_FM_EREPORT_RDMC_RBRFULL: 46093859Sml29623 case NXGE_FM_EREPORT_RDMC_RBRLOGPAGE: 46103859Sml29623 case NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE: 46113859Sml29623 case NXGE_FM_EREPORT_RDMC_CONFIG_ERR: 46123859Sml29623 RXDMA_REG_READ64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG, 46136929Smisaki chan, &cs.value); 46143859Sml29623 if (err_id == NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR) 46153859Sml29623 cs.bits.hdw.rcr_ack_err = 1; 46163859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR) 46173859Sml29623 cs.bits.hdw.dc_fifo_err = 1; 46183859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR) 46193859Sml29623 cs.bits.hdw.rcr_sha_par = 1; 46203859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR) 46213859Sml29623 cs.bits.hdw.rbr_pre_par = 1; 46223859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_TMOUT) 46233859Sml29623 cs.bits.hdw.rbr_tmout = 1; 46243859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR) 46253859Sml29623 cs.bits.hdw.rsp_cnt_err = 1; 46263859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS) 46273859Sml29623 cs.bits.hdw.byte_en_bus = 1; 46283859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR) 46293859Sml29623 cs.bits.hdw.rsp_dat_err = 1; 46303859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_CONFIG_ERR) 46313859Sml29623 cs.bits.hdw.config_err = 1; 46323859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RCRINCON) 46333859Sml29623 cs.bits.hdw.rcrincon = 1; 46343859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RCRFULL) 46353859Sml29623 cs.bits.hdw.rcrfull = 1; 46363859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RBRFULL) 46373859Sml29623 cs.bits.hdw.rbrfull = 1; 46383859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_RBRLOGPAGE) 46393859Sml29623 cs.bits.hdw.rbrlogpage = 1; 46403859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE) 46413859Sml29623 cs.bits.hdw.cfiglogpage = 1; 46425125Sjoycey #if defined(__i386) 46435125Sjoycey cmn_err(CE_NOTE, "!Write 0x%llx to RX_DMA_CTL_STAT_DBG_REG\n", 46446929Smisaki cs.value); 46455125Sjoycey #else 46463859Sml29623 cmn_err(CE_NOTE, "!Write 0x%lx to RX_DMA_CTL_STAT_DBG_REG\n", 46476929Smisaki cs.value); 46485125Sjoycey #endif 46493859Sml29623 RXDMA_REG_WRITE64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG, 46506929Smisaki chan, cs.value); 46513859Sml29623 break; 46523859Sml29623 case NXGE_FM_EREPORT_RDMC_ID_MISMATCH: 46533859Sml29623 case NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR: 46543859Sml29623 case NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR: 46553859Sml29623 cdfs.value = 0; 46563859Sml29623 if (err_id == NXGE_FM_EREPORT_RDMC_ID_MISMATCH) 46573859Sml29623 cdfs.bits.ldw.id_mismatch = (1 << nxgep->mac.portnum); 46583859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR) 46593859Sml29623 cdfs.bits.ldw.zcp_eop_err = (1 << nxgep->mac.portnum); 46603859Sml29623 else if (err_id == NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR) 46613859Sml29623 cdfs.bits.ldw.ipp_eop_err = (1 << nxgep->mac.portnum); 46625125Sjoycey #if defined(__i386) 46635125Sjoycey cmn_err(CE_NOTE, 46646929Smisaki "!Write 0x%llx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n", 46656929Smisaki cdfs.value); 46665125Sjoycey #else 46673859Sml29623 cmn_err(CE_NOTE, 46686929Smisaki "!Write 0x%lx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n", 46696929Smisaki cdfs.value); 46705125Sjoycey #endif 46716495Sspeer NXGE_REG_WR64(nxgep->npi_handle, 46726495Sspeer RX_CTL_DAT_FIFO_STAT_DBG_REG, cdfs.value); 46733859Sml29623 break; 46743859Sml29623 case NXGE_FM_EREPORT_RDMC_DCF_ERR: 46753859Sml29623 break; 46765165Syc148097 case NXGE_FM_EREPORT_RDMC_RCR_ERR: 46773859Sml29623 break; 46783859Sml29623 } 46793859Sml29623 } 46806495Sspeer 46816495Sspeer static void 46826495Sspeer nxge_rxdma_databuf_free(p_rx_rbr_ring_t rbr_p) 46836495Sspeer { 46846495Sspeer rxring_info_t *ring_info; 46856495Sspeer int index; 46866495Sspeer uint32_t chunk_size; 46876495Sspeer uint64_t kaddr; 46886495Sspeer uint_t num_blocks; 46896495Sspeer 46906495Sspeer NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_rxdma_databuf_free")); 46916495Sspeer 46926495Sspeer if (rbr_p == NULL) { 46936495Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 46946495Sspeer "==> nxge_rxdma_databuf_free: NULL rbr pointer")); 46956495Sspeer return; 46966495Sspeer } 46976495Sspeer 46986495Sspeer if (rbr_p->rbr_alloc_type == DDI_MEM_ALLOC) { 46996495Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 47006495Sspeer "==> nxge_rxdma_databuf_free: DDI")); 47016495Sspeer return; 47026495Sspeer } 47036495Sspeer 47046495Sspeer ring_info = rbr_p->ring_info; 47056495Sspeer if (ring_info == NULL) { 47066495Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 47076495Sspeer "==> nxge_rxdma_databuf_free: NULL ring info")); 47086495Sspeer return; 47096495Sspeer } 47106495Sspeer num_blocks = rbr_p->num_blocks; 47116495Sspeer for (index = 0; index < num_blocks; index++) { 47126495Sspeer kaddr = ring_info->buffer[index].kaddr; 47136495Sspeer chunk_size = ring_info->buffer[index].buf_size; 47146495Sspeer NXGE_DEBUG_MSG((NULL, DMA_CTL, 47156495Sspeer "==> nxge_rxdma_databuf_free: free chunk %d " 47166495Sspeer "kaddrp $%p chunk size %d", 47176495Sspeer index, kaddr, chunk_size)); 47186495Sspeer if (kaddr == NULL) continue; 47196495Sspeer nxge_free_buf(rbr_p->rbr_alloc_type, kaddr, chunk_size); 47206495Sspeer ring_info->buffer[index].kaddr = NULL; 47216495Sspeer } 47226495Sspeer 47236495Sspeer NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_rxdma_databuf_free")); 47246495Sspeer } 47256495Sspeer 47266495Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 47276495Sspeer extern void contig_mem_free(void *, size_t); 47286495Sspeer #endif 47296495Sspeer 47306495Sspeer void 47316495Sspeer nxge_free_buf(buf_alloc_type_t alloc_type, uint64_t kaddr, uint32_t buf_size) 47326495Sspeer { 47336495Sspeer NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_free_buf")); 47346495Sspeer 47356495Sspeer if (kaddr == NULL || !buf_size) { 47366495Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 47376495Sspeer "==> nxge_free_buf: invalid kaddr $%p size to free %d", 47386495Sspeer kaddr, buf_size)); 47396495Sspeer return; 47406495Sspeer } 47416495Sspeer 47426495Sspeer switch (alloc_type) { 47436495Sspeer case KMEM_ALLOC: 47446495Sspeer NXGE_DEBUG_MSG((NULL, DMA_CTL, 47456495Sspeer "==> nxge_free_buf: freeing kmem $%p size %d", 47466495Sspeer kaddr, buf_size)); 47476495Sspeer #if defined(__i386) 47486495Sspeer KMEM_FREE((void *)(uint32_t)kaddr, buf_size); 47496495Sspeer #else 47506495Sspeer KMEM_FREE((void *)kaddr, buf_size); 47516495Sspeer #endif 47526495Sspeer break; 47536495Sspeer 47546495Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND) 47556495Sspeer case CONTIG_MEM_ALLOC: 47566495Sspeer NXGE_DEBUG_MSG((NULL, DMA_CTL, 47576495Sspeer "==> nxge_free_buf: freeing contig_mem kaddr $%p size %d", 47586495Sspeer kaddr, buf_size)); 47596495Sspeer contig_mem_free((void *)kaddr, buf_size); 47606495Sspeer break; 47616495Sspeer #endif 47626495Sspeer 47636495Sspeer default: 47646495Sspeer NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, 47656495Sspeer "<== nxge_free_buf: unsupported alloc type %d", 47666495Sspeer alloc_type)); 47676495Sspeer return; 47686495Sspeer } 47696495Sspeer 47706495Sspeer NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_free_buf")); 47716495Sspeer } 4772