1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 #include <rte_mbuf.h> 11 #include <rte_mempool.h> 12 #include <rte_prefetch.h> 13 #include <rte_vect.h> 14 15 #include <mlx5_glue.h> 16 #include <mlx5_prm.h> 17 18 #include "mlx5_defs.h" 19 #include "mlx5.h" 20 #include "mlx5_utils.h" 21 #include "mlx5_rxtx.h" 22 #include "mlx5_rxtx_vec.h" 23 #include "mlx5_autoconf.h" 24 25 #if defined RTE_ARCH_X86_64 26 #include "mlx5_rxtx_vec_sse.h" 27 #elif defined RTE_ARCH_ARM64 28 #include "mlx5_rxtx_vec_neon.h" 29 #elif defined RTE_ARCH_PPC_64 30 #include "mlx5_rxtx_vec_altivec.h" 31 #else 32 #error "This should not be compiled if SIMD instructions are not supported." 33 #endif 34 35 /** 36 * Skip error packets. 37 * 38 * @param rxq 39 * Pointer to RX queue structure. 40 * @param[out] pkts 41 * Array to store received packets. 42 * @param pkts_n 43 * Maximum number of packets in array. 44 * 45 * @return 46 * Number of packets successfully received (<= pkts_n). 47 */ 48 static uint16_t 49 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 50 uint16_t pkts_n) 51 { 52 uint16_t n = 0; 53 unsigned int i; 54 #ifdef MLX5_PMD_SOFT_COUNTERS 55 uint32_t err_bytes = 0; 56 #endif 57 58 for (i = 0; i < pkts_n; ++i) { 59 struct rte_mbuf *pkt = pkts[i]; 60 61 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 62 #ifdef MLX5_PMD_SOFT_COUNTERS 63 err_bytes += PKT_LEN(pkt); 64 #endif 65 rte_pktmbuf_free_seg(pkt); 66 } else { 67 pkts[n++] = pkt; 68 } 69 } 70 rxq->stats.idropped += (pkts_n - n); 71 #ifdef MLX5_PMD_SOFT_COUNTERS 72 /* Correct counters of errored completions. */ 73 rxq->stats.ipackets -= (pkts_n - n); 74 rxq->stats.ibytes -= err_bytes; 75 #endif 76 mlx5_rx_err_handle(rxq, 1); 77 return n; 78 } 79 80 /** 81 * DPDK callback for vectorized RX. 82 * 83 * @param dpdk_rxq 84 * Generic pointer to RX queue structure. 85 * @param[out] pkts 86 * Array to store received packets. 87 * @param pkts_n 88 * Maximum number of packets in array. 89 * 90 * @return 91 * Number of packets successfully received (<= pkts_n). 92 */ 93 uint16_t 94 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 95 { 96 struct mlx5_rxq_data *rxq = dpdk_rxq; 97 uint16_t nb_rx = 0; 98 uint16_t tn = 0; 99 uint64_t err = 0; 100 bool no_cq = false; 101 102 do { 103 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, &err, &no_cq); 104 if (unlikely(err | rxq->err_state)) 105 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 106 tn += nb_rx; 107 if (unlikely(no_cq)) 108 break; 109 } while (tn != pkts_n); 110 return tn; 111 } 112 113 /** 114 * Check a RX queue can support vectorized RX. 115 * 116 * @param rxq 117 * Pointer to RX queue. 118 * 119 * @return 120 * 1 if supported, negative errno value if not. 121 */ 122 int __rte_cold 123 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 124 { 125 struct mlx5_rxq_ctrl *ctrl = 126 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 127 128 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) 129 return -ENOTSUP; 130 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 131 return -ENOTSUP; 132 if (rxq->lro) 133 return -ENOTSUP; 134 return 1; 135 } 136 137 /** 138 * Check a device can support vectorized RX. 139 * 140 * @param dev 141 * Pointer to Ethernet device. 142 * 143 * @return 144 * 1 if supported, negative errno value if not. 145 */ 146 int __rte_cold 147 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 148 { 149 struct mlx5_priv *priv = dev->data->dev_private; 150 uint32_t i; 151 152 if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) 153 return -ENOTSUP; 154 if (!priv->config.rx_vec_en) 155 return -ENOTSUP; 156 if (mlx5_mprq_enabled(dev)) 157 return -ENOTSUP; 158 /* All the configured queues should support. */ 159 for (i = 0; i < priv->rxqs_n; ++i) { 160 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 161 162 if (!rxq) 163 continue; 164 if (mlx5_rxq_check_vec_support(rxq) < 0) 165 break; 166 } 167 if (i != priv->rxqs_n) 168 return -ENOTSUP; 169 return 1; 170 } 171