127 #define GE_RXDSYNC(sc, rxq, n, ops) \  argument
128 	bus_dmamap_sync((sc)->sc_dmat, (rxq)->rxq_desc_mem.gdm_map, \
129 	    (n) * sizeof((rxq)->rxq_descs[0]), sizeof((rxq)->rxq_descs[0]), \
131 #define	GE_RXDPRESYNC(sc, rxq, n) \  argument
132 	GE_RXDSYNC(sc, rxq, n, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)
133 #define	GE_RXDPOSTSYNC(sc, rxq, n) \  argument
134 	GE_RXDSYNC(sc, rxq, n, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)
758 	struct gfe_rxqueue * const rxq = &sc->sc_rxq[rxprio];  in gfe_rx_rxqalloc()  local
764 	error = gfe_dmamem_alloc(sc, &rxq->rxq_desc_mem, 1,  in gfe_rx_rxqalloc()
771 	error = gfe_dmamem_alloc(sc, &rxq->rxq_buf_mem, GE_RXBUF_NSEGS,  in gfe_rx_rxqalloc()
784 	struct gfe_rxqueue * const rxq = &sc->sc_rxq[rxprio];  in gfe_rx_rxqinit()  local
801 		KASSERT(rxq->rxq_desc_mem.gdm_kva != NULL);  in gfe_rx_rxqinit()
802 		KASSERT(rxq->rxq_buf_mem.gdm_kva != NULL);  in gfe_rx_rxqinit()
805 	memset(rxq->rxq_desc_mem.gdm_kva, 0, GE_RXDESC_MEMSIZE);  in gfe_rx_rxqinit()
807 	rxq->rxq_descs =  in gfe_rx_rxqinit()
808 	    (volatile struct gt_eth_desc *) rxq->rxq_desc_mem.gdm_kva;  in gfe_rx_rxqinit()
809 	rxq->rxq_desc_busaddr = rxq->rxq_desc_mem.gdm_map->dm_segs[0].ds_addr;  in gfe_rx_rxqinit()
810 	rxq->rxq_bufs = (struct gfe_rxbuf *) rxq->rxq_buf_mem.gdm_kva;  in gfe_rx_rxqinit()
811 	rxq->rxq_fi = 0;  in gfe_rx_rxqinit()
812 	rxq->rxq_active = GE_RXDESC_MAX;  in gfe_rx_rxqinit()
814 	ds = rxq->rxq_buf_mem.gdm_map->dm_segs;  in gfe_rx_rxqinit()
815 	nxtaddr = rxq->rxq_desc_busaddr + sizeof(*rxd);  in gfe_rx_rxqinit()
816 	for (idx = 0, rxd = rxq->rxq_descs; idx < GE_RXDESC_MAX;  in gfe_rx_rxqinit()
825 			nxtaddr = rxq->rxq_desc_busaddr;  in gfe_rx_rxqinit()
833 	bus_dmamap_sync(sc->sc_dmat, rxq->rxq_desc_mem.gdm_map, 0,  in gfe_rx_rxqinit()
834 			rxq->rxq_desc_mem.gdm_map->dm_mapsize,  in gfe_rx_rxqinit()
836 	bus_dmamap_sync(sc->sc_dmat, rxq->rxq_buf_mem.gdm_map, 0,  in gfe_rx_rxqinit()
837 			rxq->rxq_buf_mem.gdm_map->dm_mapsize,  in gfe_rx_rxqinit()
840 	rxq->rxq_intrbits = ETH_IR_RxBuffer | ETH_IR_RxError;  in gfe_rx_rxqinit()
843 		rxq->rxq_intrbits |= ETH_IR_RxBuffer_3 | ETH_IR_RxError_3;  in gfe_rx_rxqinit()
844 		rxq->rxq_efrdp = ETH_EFRDP3;  in gfe_rx_rxqinit()
845 		rxq->rxq_ecrdp = ETH_ECRDP3;  in gfe_rx_rxqinit()
848 		rxq->rxq_intrbits |= ETH_IR_RxBuffer_2 | ETH_IR_RxError_2;  in gfe_rx_rxqinit()
849 		rxq->rxq_efrdp = ETH_EFRDP2;  in gfe_rx_rxqinit()
850 		rxq->rxq_ecrdp = ETH_ECRDP2;  in gfe_rx_rxqinit()
853 		rxq->rxq_intrbits |= ETH_IR_RxBuffer_1 | ETH_IR_RxError_1;  in gfe_rx_rxqinit()
854 		rxq->rxq_efrdp = ETH_EFRDP1;  in gfe_rx_rxqinit()
855 		rxq->rxq_ecrdp = ETH_ECRDP1;  in gfe_rx_rxqinit()
858 		rxq->rxq_intrbits |= ETH_IR_RxBuffer_0 | ETH_IR_RxError_0;  in gfe_rx_rxqinit()
859 		rxq->rxq_efrdp = ETH_EFRDP0;  in gfe_rx_rxqinit()
860 		rxq->rxq_ecrdp = ETH_ECRDP0;  in gfe_rx_rxqinit()
871 	struct gfe_rxqueue * const rxq = &sc->sc_rxq[rxprio];  in gfe_rx_get()  local
872 	struct mbuf *m = rxq->rxq_curpkt;  in gfe_rx_get()
877 	while (rxq->rxq_active > 0) {  in gfe_rx_get()
878 		volatile struct gt_eth_desc *rxd = &rxq->rxq_descs[rxq->rxq_fi];  in gfe_rx_get()
879 		struct gfe_rxbuf *rxb = &rxq->rxq_bufs[rxq->rxq_fi];  in gfe_rx_get()
884 		GE_RXDPOSTSYNC(sc, rxq, rxq->rxq_fi);  in gfe_rx_get()
886 		GE_DPRINTF(sc, (":%d=%#x", rxq->rxq_fi, cmdsts));  in gfe_rx_get()
887 		rxq->rxq_cmdsts = cmdsts;  in gfe_rx_get()
895 			GE_RXDPRESYNC(sc, rxq, rxq->rxq_fi);  in gfe_rx_get()
908 			--rxq->rxq_active;  in gfe_rx_get()
935 		rxq->rxq_cmdsts = cmdsts;  in gfe_rx_get()
936 		--rxq->rxq_active;  in gfe_rx_get()
938 		bus_dmamap_sync(sc->sc_dmat, rxq->rxq_buf_mem.gdm_map,  in gfe_rx_get()
939 		    rxq->rxq_fi * sizeof(*rxb), buflen, BUS_DMASYNC_POSTREAD);  in gfe_rx_get()
948 		    (rxq->rxq_cmdsts & RX_STS_M) == 0 ||  in gfe_rx_get()
949 		    (rxq->rxq_cmdsts & RX_STS_HE) ||  in gfe_rx_get()
961 		rxq->rxq_cmdsts = 0;  in gfe_rx_get()
968 		    rxq->rxq_fi,  in gfe_rx_get()
972 		GE_RXDPRESYNC(sc, rxq, rxq->rxq_fi);  in gfe_rx_get()
973 		if (++rxq->rxq_fi == GE_RXDESC_MAX)  in gfe_rx_get()
974 			rxq->rxq_fi = 0;  in gfe_rx_get()
975 		rxq->rxq_active++;  in gfe_rx_get()
977 	rxq->rxq_curpkt = m;  in gfe_rx_get()
985 	struct gfe_rxqueue *rxq;  in gfe_rx_process()  local
1004 		rxq = &sc->sc_rxq[rxprio];  in gfe_rx_process()
1005 		sc->sc_idlemask |= (rxq->rxq_intrbits & ETH_IR_RxBits);  in gfe_rx_process()
1006 		intrmask &= ~(rxq->rxq_intrbits & ETH_IR_RxBits);  in gfe_rx_process()
1013 		    device_xname(sc->sc_dev), rxprio, rxq->rxq_fi));  in gfe_rx_process()
1015 		bus_dmamap_sync(sc->sc_dmat, rxq->rxq_desc_mem.gdm_map,  in gfe_rx_process()
1016 		    0, rxq->rxq_desc_mem.gdm_size,  in gfe_rx_process()
1019 			volatile struct gt_eth_desc *rxd = &rxq->rxq_descs[idx];  in gfe_rx_process()
1024 		bus_dmamap_sync(sc->sc_dmat, rxq->rxq_desc_mem.gdm_map,  in gfe_rx_process()
1025 		    0, rxq->rxq_desc_mem.gdm_size,  in gfe_rx_process()
1029 		    device_xname(sc->sc_dev), rxprio, rxq->rxq_fi,  in gfe_rx_process()
1030 		    rxq->rxq_cmdsts, masks[0], masks[1]);  in gfe_rx_process()
1043 	struct gfe_rxqueue *rxq;  in gfe_rx_prime()  local
1051 	rxq = &sc->sc_rxq[GE_RXPRIO_HI];  in gfe_rx_prime()
1053 		GE_WRITE(sc, ETH_EFRDP3, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1054 		GE_WRITE(sc, ETH_ECRDP3, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1056 	sc->sc_intrmask |= rxq->rxq_intrbits;  in gfe_rx_prime()
1062 		rxq = &sc->sc_rxq[GE_RXPRIO_MEDHI];  in gfe_rx_prime()
1063 		GE_WRITE(sc, ETH_EFRDP2, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1064 		GE_WRITE(sc, ETH_ECRDP2, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1065 		sc->sc_intrmask |= rxq->rxq_intrbits;  in gfe_rx_prime()
1072 		rxq = &sc->sc_rxq[GE_RXPRIO_MEDLO];  in gfe_rx_prime()
1073 		GE_WRITE(sc, ETH_EFRDP1, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1074 		GE_WRITE(sc, ETH_ECRDP1, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1075 		sc->sc_intrmask |= rxq->rxq_intrbits;  in gfe_rx_prime()
1082 		rxq = &sc->sc_rxq[GE_RXPRIO_LO];  in gfe_rx_prime()
1083 		GE_WRITE(sc, ETH_EFRDP0, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1084 		GE_WRITE(sc, ETH_ECRDP0, rxq->rxq_desc_busaddr);  in gfe_rx_prime()
1085 		sc->sc_intrmask |= rxq->rxq_intrbits;  in gfe_rx_prime()
1096 	struct gfe_rxqueue *rxq = &sc->sc_rxq[rxprio];  in gfe_rx_cleanup()  local
1098 	if (rxq == NULL) {  in gfe_rx_cleanup()
1103 	m_freem(rxq->rxq_curpkt);  in gfe_rx_cleanup()
1105 		gfe_dmamem_free(sc, &rxq->rxq_desc_mem);  in gfe_rx_cleanup()
1106 		gfe_dmamem_free(sc, &rxq->rxq_buf_mem);  in gfe_rx_cleanup()
1150 			struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_HI];  in gfe_tick()  local
1151 			rxq->rxq_fi = 0;  in gfe_tick()
1152 			GE_WRITE(sc, ETH_EFRDP3, rxq->rxq_desc_busaddr);  in gfe_tick()
1153 			GE_WRITE(sc, ETH_ECRDP3, rxq->rxq_desc_busaddr);  in gfe_tick()
1156 			struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_MEDHI];  in gfe_tick()  local
1157 			rxq->rxq_fi = 0;  in gfe_tick()
1158 			GE_WRITE(sc, ETH_EFRDP2, rxq->rxq_desc_busaddr);  in gfe_tick()
1159 			GE_WRITE(sc, ETH_ECRDP2, rxq->rxq_desc_busaddr);  in gfe_tick()
1162 			struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_MEDLO];  in gfe_tick()  local
1163 			rxq->rxq_fi = 0;  in gfe_tick()
1164 			GE_WRITE(sc, ETH_EFRDP1, rxq->rxq_desc_busaddr);  in gfe_tick()
1165 			GE_WRITE(sc, ETH_ECRDP1, rxq->rxq_desc_busaddr);  in gfe_tick()
1168 			struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_LO];  in gfe_tick()  local
1169 			rxq->rxq_fi = 0;  in gfe_tick()
1170 			GE_WRITE(sc, ETH_EFRDP0, rxq->rxq_desc_busaddr);  in gfe_tick()
1171 			GE_WRITE(sc, ETH_ECRDP0, rxq->rxq_desc_busaddr);  in gfe_tick()