1 /* SPDX-License-Identifier: BSD-3-Clause */
34  * @brief iflib Tx/Rx hotpath
36  * Main location for the iflib Tx/Rx hotpath implementation.
44 /* Tx/Rx hotpath utility functions */
78 /* Macro to help extract the NIC mode flexible Rx descriptor fields from the
79  * advanced 32byte Rx descriptors.
82 	(((struct ice_32b_rx_flex_desc_nic *)desc)->field)
86  * @brief Tx/Rx operations for the iflib stack
88  * Structure defining the Tx and Rx related operations that iflib can request
105  * @brief Tx/Rx operations for the iflib stack, for subinterfaces
107  * Structure defining the Tx and Rx related operations that iflib can request
123  * _ice_ift_txd_encap - prepare Tx descriptors for a packet
124  * @txq: driver's TX queue context
132  * Return 0 on success, non-zero error code on failure.
137 	int nsegs = pi->ipi_nsegs;  in _ice_ift_txd_encap()
138 	bus_dma_segment_t *segs = pi->ipi_segs;  in _ice_ift_txd_encap()
144 	i = pi->ipi_pidx;  in _ice_ift_txd_encap()
147 	if (pi->ipi_csum_flags & ICE_CSUM_OFFLOAD) {  in _ice_ift_txd_encap()
149 		if (pi->ipi_csum_flags & CSUM_TSO) {  in _ice_ift_txd_encap()
156 	if (pi->ipi_mflags & M_VLANTAG)  in _ice_ift_txd_encap()
159 	mask = txq->desc_count - 1;  in _ice_ift_txd_encap()
163 		txd = &txq->tx_base[i];  in _ice_ift_txd_encap()
166 		txd->buf_addr = htole64(segs[j].ds_addr);  in _ice_ift_txd_encap()
167 		txd->cmd_type_offset_bsz =  in _ice_ift_txd_encap()
172 		    | ((u64)htole16(pi->ipi_vtag) << ICE_TXD_QW1_L2TAG1_S));  in _ice_ift_txd_encap()
174 		txq->stats.tx_bytes += seglen;  in _ice_ift_txd_encap()
181 	txd->cmd_type_offset_bsz |=  in _ice_ift_txd_encap()
185 	txq->tx_rsq[txq->tx_rs_pidx] = pidx_last;  in _ice_ift_txd_encap()
186 	txq->tx_rs_pidx = (txq->tx_rs_pidx+1) & mask;  in _ice_ift_txd_encap()
187 	MPASS(txq->tx_rs_pidx != txq->tx_rs_cidx);  in _ice_ift_txd_encap()
189 	pi->ipi_new_pidx = i;  in _ice_ift_txd_encap()
191 	++txq->stats.tx_packets;  in _ice_ift_txd_encap()
196  * ice_ift_txd_encap - prepare Tx descriptors for a packet
205  * Return 0 on success, non-zero error code on failure.
211 	struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[pi->ipi_qsidx];  in ice_ift_txd_encap()
217  * ice_ift_txd_flush - Flush Tx descriptors to hardware
219  * @txqid: the Tx queue to flush
229 	struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];  in ice_ift_txd_flush()
230 	struct ice_hw *hw = &sc->hw;  in ice_ift_txd_flush()
232 	wr32(hw, txq->tail, pidx);  in ice_ift_txd_flush()
236  * _ice_ift_txd_credits_update - cleanup Tx descriptors
238  * @txq: the Tx queue to update
257 	rs_cidx = txq->tx_rs_cidx;  in _ice_ift_txd_credits_update()
258 	if (rs_cidx == txq->tx_rs_pidx)  in _ice_ift_txd_credits_update()
260 	cur = txq->tx_rsq[rs_cidx];  in _ice_ift_txd_credits_update()
262 	is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);  in _ice_ift_txd_credits_update()
269 	prev = txq->tx_cidx_processed;  in _ice_ift_txd_credits_update()
270 	ntxd = txq->desc_count;  in _ice_ift_txd_credits_update()
273 		delta = (int32_t)cur - (int32_t)prev;  in _ice_ift_txd_credits_update()
279 		rs_cidx = (rs_cidx + 1) & (ntxd-1);  in _ice_ift_txd_credits_update()
280 		if (rs_cidx == txq->tx_rs_pidx)  in _ice_ift_txd_credits_update()
282 		cur = txq->tx_rsq[rs_cidx];  in _ice_ift_txd_credits_update()
284 		is_done = ice_is_tx_desc_done(&txq->tx_base[cur]);  in _ice_ift_txd_credits_update()
287 	txq->tx_rs_cidx = rs_cidx;  in _ice_ift_txd_credits_update()
288 	txq->tx_cidx_processed = prev;  in _ice_ift_txd_credits_update()
294  * ice_ift_txd_credits_update - cleanup PF VSI Tx descriptors
296  * @txqid: the Tx queue to update
308 	struct ice_tx_queue *txq = &sc->pf_vsi.tx_queues[txqid];  in ice_ift_txd_credits_update()
314  * _ice_ift_rxd_available - Return number of available Rx packets
315  * @rxq: RX queue driver structure
317  * @budget: maximum Rx budget
319  * Determines how many Rx packets are available on the queue, up to a maximum
329 	nrxd = rxq->desc_count;  in _ice_ift_rxd_available()
331 	for (cnt = 0, i = pidx; cnt < nrxd - 1 && cnt < budget;) {  in _ice_ift_rxd_available()
332 		rxd = &rxq->rx_base[i];  in _ice_ift_rxd_available()
333 		status0 = le16toh(rxd->wb.status_error0);  in _ice_ift_rxd_available()
347  * ice_ift_rxd_available - Return number of available Rx packets
349  * @rxqid: the Rx queue id
351  * @budget: maximum Rx budget
354  * that iflib requires for RX processing.
360 	struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];  in ice_ift_rxd_available()
366  * ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer
371  * used by iflib for RX packet processing.
377 	struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[ri->iri_qsidx];  in ice_ift_rxd_pkt_get()
383  * _ice_ift_rxd_pkt_get - Called by iflib to send data to upper layer
384  * @rxq: RX queue driver structure
400 	cidx = ri->iri_cidx;  in _ice_ift_rxd_pkt_get()
406 		cur = &rxq->rx_base[cidx];  in _ice_ift_rxd_pkt_get()
407 		status0 = le16toh(cur->wb.status_error0);  in _ice_ift_rxd_pkt_get()
408 		plen = le16toh(cur->wb.pkt_len) &  in _ice_ift_rxd_pkt_get()
414 		ri->iri_len += plen;  in _ice_ift_rxd_pkt_get()
416 		cur->wb.status_error0 = 0;  in _ice_ift_rxd_pkt_get()
419 		ri->iri_frags[i].irf_flid = 0;  in _ice_ift_rxd_pkt_get()
420 		ri->iri_frags[i].irf_idx = cidx;  in _ice_ift_rxd_pkt_get()
421 		ri->iri_frags[i].irf_len = plen;  in _ice_ift_rxd_pkt_get()
422 		if (++cidx == rxq->desc_count)  in _ice_ift_rxd_pkt_get()
433 		rxq->stats.desc_errs++;  in _ice_ift_rxd_pkt_get()
439 		ri->iri_vtag = le16toh(cur->wb.l2tag1);  in _ice_ift_rxd_pkt_get()
440 		ri->iri_flags |= M_VLANTAG;  in _ice_ift_rxd_pkt_get()
443 	/* Capture soft statistics for this Rx queue */  in _ice_ift_rxd_pkt_get()
444 	rxq->stats.rx_packets++;  in _ice_ift_rxd_pkt_get()
445 	rxq->stats.rx_bytes += ri->iri_len;  in _ice_ift_rxd_pkt_get()
448 	ptype = le16toh(cur->wb.ptype_flex_flags0) &  in _ice_ift_rxd_pkt_get()
450 	if ((if_getcapenable(ri->iri_ifp) & IFCAP_RXCSUM) != 0)  in _ice_ift_rxd_pkt_get()
451 		ice_rx_checksum(rxq, &ri->iri_csum_flags,  in _ice_ift_rxd_pkt_get()
452 				&ri->iri_csum_data, status0, ptype);  in _ice_ift_rxd_pkt_get()
454 	/* Set remaining iflib RX descriptor info fields */  in _ice_ift_rxd_pkt_get()
455 	ri->iri_flowid = le32toh(RX_FLEX_NIC(&cur->wb, rss_hash));  in _ice_ift_rxd_pkt_get()
456 	ri->iri_rsstype = ice_ptype_to_hash(ptype);  in _ice_ift_rxd_pkt_get()
457 	ri->iri_nfrags = i;  in _ice_ift_rxd_pkt_get()
462  * ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware
464  * @iru: the Rx descriptor update structure
467  * used by iflib for RX packet processing.
478 	paddrs = iru->iru_paddrs;  in ice_ift_rxd_refill()
479 	pidx = iru->iru_pidx;  in ice_ift_rxd_refill()
480 	qsidx = iru->iru_qsidx;  in ice_ift_rxd_refill()
481 	count = iru->iru_count;  in ice_ift_rxd_refill()
483 	rxq = &(sc->pf_vsi.rx_queues[qsidx]);  in ice_ift_rxd_refill()
489  * _ice_ift_rxd_refill - Prepare Rx descriptors for re-use by hardware
490  * @rxq: RX queue driver structure
495  * Update the Rx descriptor indices for a given queue, assigning new physical
496  * addresses to the descriptors, preparing them for re-use by the hardware.
506 		rxq->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]);  in _ice_ift_rxd_refill()
507 		if (++next_pidx == (uint32_t)rxq->desc_count)  in _ice_ift_rxd_refill()
513  * ice_ift_rxd_flush - Flush Rx descriptors to hardware
515  * @rxqid: the Rx queue to flush
520  * used by iflib for RX packet processing.
527 	struct ice_rx_queue *rxq = &sc->pf_vsi.rx_queues[rxqid];  in ice_ift_rxd_flush()
533  * _ice_ift_rxd_flush - Flush Rx descriptors to hardware
535  * @rxq: RX queue driver structure
544 	wr32(&sc->hw, rxq->tail, pidx);  in _ice_ift_rxd_flush()
548  * ice_ift_queue_select - Select queue index to transmit packet on
553  * Called by iflib to determine which queue index to transmit the packet
555  * queue index for the right transmit class when multiple traffic classes are
563 	struct ice_vsi *vsi = &sc->pf_vsi;  in ice_ift_queue_select()
569 	/* Only go out on default queue if ALTQ is enabled */  in ice_ift_queue_select()
570 	struct ifnet *ifp = (struct ifnet *)iflib_get_ifp(sc->ctx);  in ice_ift_queue_select()
575 	if (!ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS)) {  in ice_ift_queue_select()
577 			/* Default iflib queue selection method */  in ice_ift_queue_select()
578 			return (m->m_pkthdr.flowid % sc->pf_vsi.num_tx_queues);  in ice_ift_queue_select()
586 	local_dcbx_cfg = &sc->hw.port_info->qos_cfg.local_dcbx_cfg;  in ice_ift_queue_select()
589 	if ((local_dcbx_cfg->pfc_mode == ICE_QOS_MODE_DSCP) &&  in ice_ift_queue_select()
590 	    (pi->ipi_flags & (IPI_TX_IPV4 | IPI_TX_IPV6))) {  in ice_ift_queue_select()
591 		u8 dscp_val = pi->ipi_ip_tos >> 2;  in ice_ift_queue_select()
592 		tc = local_dcbx_cfg->dscp_map[dscp_val];  in ice_ift_queue_select()
595 	if (m->m_flags & M_VLANTAG) { /* ICE_QOS_MODE_VLAN */  in ice_ift_queue_select()
596 		up = EVL_PRIOFTAG(m->m_pkthdr.ether_vtag);  in ice_ift_queue_select()
597 		tc = local_dcbx_cfg->etscfg.prio_table[up];  in ice_ift_queue_select()
600 	tc_base_queue = vsi->tc_info[tc].qoffset;  in ice_ift_queue_select()
601 	tc_qcount = vsi->tc_info[tc].qcount_tx;  in ice_ift_queue_select()
604 		return ((m->m_pkthdr.flowid % tc_qcount) + tc_base_queue);  in ice_ift_queue_select()
610  * ice_ift_txd_credits_update_subif - cleanup subinterface VSI Tx descriptors
612  * @txqid: the Tx queue to update
624 	struct ice_softc *sc = mif->back;  in ice_ift_txd_credits_update_subif()
625 	struct ice_tx_queue *txq = &mif->vsi->tx_queues[txqid];  in ice_ift_txd_credits_update_subif()
631  * ice_ift_txd_encap_subif - prepare Tx descriptors for a packet
644 	struct ice_tx_queue *txq = &mif->vsi->tx_queues[pi->ipi_qsidx];  in ice_ift_txd_encap_subif()
650  * ice_ift_txd_flush_subif - Flush Tx descriptors to hardware
652  * @txqid: the Tx queue to flush
657  * pointer to iflib for use with non-main-PF VSI TX queues.
663 	struct ice_tx_queue *txq = &mif->vsi->tx_queues[txqid];  in ice_ift_txd_flush_subif()
664 	struct ice_hw *hw = &mif->back->hw;  in ice_ift_txd_flush_subif()
666 	wr32(hw, txq->tail, pidx);  in ice_ift_txd_flush_subif()
670  * ice_ift_rxd_available_subif - Return number of available Rx packets
672  * @rxqid: the Rx queue id
674  * @budget: maximum Rx budget
676  * Determines how many Rx packets are available on the queue, up to a maximum
685 	struct ice_rx_queue *rxq = &mif->vsi->rx_queues[rxqid];  in ice_ift_rxd_available_subif()
691  * ice_ift_rxd_pkt_get_subif - Called by iflib to send data to upper layer
696  * used by iflib for RX packet processing, for iflib subinterfaces.
702 	struct ice_rx_queue *rxq = &mif->vsi->rx_queues[ri->iri_qsidx];  in ice_ift_rxd_pkt_get_subif()
708  * ice_ift_rxd_refill_subif - Prepare Rx descriptors for re-use by hardware
710  * @iru: the Rx descriptor update structure
713  * used by iflib for RX packet processing, for iflib subinterfaces.
719 	struct ice_rx_queue *rxq = &mif->vsi->rx_queues[iru->iru_qsidx];  in ice_ift_rxd_refill_subif()
725 	paddrs = iru->iru_paddrs;  in ice_ift_rxd_refill_subif()
726 	pidx = iru->iru_pidx;  in ice_ift_rxd_refill_subif()
727 	count = iru->iru_count;  in ice_ift_rxd_refill_subif()
733  * ice_ift_rxd_flush_subif - Flush Rx descriptors to hardware
735  * @rxqid: the Rx queue to flush
740  * used by iflib for RX packet processing.
747 	struct ice_rx_queue *rxq = &mif->vsi->rx_queues[rxqid];  in ice_ift_rxd_flush_subif()
749 	_ice_ift_rxd_flush(mif->back, rxq, pidx);  in ice_ift_rxd_flush_subif()