1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <assert.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdlib.h> 10 11 /* Verbs header. */ 12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 13 #ifdef PEDANTIC 14 #pragma GCC diagnostic ignored "-Wpedantic" 15 #endif 16 #include <infiniband/verbs.h> 17 #include <infiniband/mlx5dv.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_mbuf.h> 23 #include <rte_mempool.h> 24 #include <rte_prefetch.h> 25 26 #include "mlx5.h" 27 #include "mlx5_utils.h" 28 #include "mlx5_rxtx.h" 29 #include "mlx5_rxtx_vec.h" 30 #include "mlx5_autoconf.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_prm.h" 33 34 #if defined RTE_ARCH_X86_64 35 #include "mlx5_rxtx_vec_sse.h" 36 #elif defined RTE_ARCH_ARM64 37 #include "mlx5_rxtx_vec_neon.h" 38 #else 39 #error "This should not be compiled if SIMD instructions are not supported." 40 #endif 41 42 /** 43 * Count the number of packets having same ol_flags and calculate cs_flags. 44 * 45 * @param pkts 46 * Pointer to array of packets. 47 * @param pkts_n 48 * Number of packets. 49 * @param cs_flags 50 * Pointer of flags to be returned. 51 * 52 * @return 53 * Number of packets having same ol_flags. 54 */ 55 static inline unsigned int 56 txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags) 57 { 58 unsigned int pos; 59 const uint64_t ol_mask = 60 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | 61 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | 62 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; 63 64 if (!pkts_n) 65 return 0; 66 /* Count the number of packets having same ol_flags. */ 67 for (pos = 1; pos < pkts_n; ++pos) 68 if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) 69 break; 70 *cs_flags = txq_ol_cksum_to_cs(pkts[0]); 71 return pos; 72 } 73 74 /** 75 * DPDK callback for vectorized TX. 76 * 77 * @param dpdk_txq 78 * Generic pointer to TX queue structure. 79 * @param[in] pkts 80 * Packets to transmit. 81 * @param pkts_n 82 * Number of packets in array. 83 * 84 * @return 85 * Number of packets successfully transmitted (<= pkts_n). 86 */ 87 uint16_t 88 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, 89 uint16_t pkts_n) 90 { 91 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 92 uint16_t nb_tx = 0; 93 94 while (pkts_n > nb_tx) { 95 uint16_t n; 96 uint16_t ret; 97 98 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 99 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0); 100 nb_tx += ret; 101 if (!ret) 102 break; 103 } 104 return nb_tx; 105 } 106 107 /** 108 * DPDK callback for vectorized TX with multi-seg packets and offload. 109 * 110 * @param dpdk_txq 111 * Generic pointer to TX queue structure. 112 * @param[in] pkts 113 * Packets to transmit. 114 * @param pkts_n 115 * Number of packets in array. 116 * 117 * @return 118 * Number of packets successfully transmitted (<= pkts_n). 119 */ 120 uint16_t 121 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) 122 { 123 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 124 uint16_t nb_tx = 0; 125 126 while (pkts_n > nb_tx) { 127 uint8_t cs_flags = 0; 128 uint16_t n; 129 uint16_t ret; 130 131 /* Transmit multi-seg packets in the head of pkts list. */ 132 if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) && 133 NB_SEGS(pkts[nb_tx]) > 1) 134 nb_tx += txq_scatter_v(txq, 135 &pkts[nb_tx], 136 pkts_n - nb_tx); 137 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 138 if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 139 n = txq_count_contig_single_seg(&pkts[nb_tx], n); 140 if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) 141 n = txq_calc_offload(&pkts[nb_tx], n, &cs_flags); 142 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); 143 nb_tx += ret; 144 if (!ret) 145 break; 146 } 147 return nb_tx; 148 } 149 150 /** 151 * Skip error packets. 152 * 153 * @param rxq 154 * Pointer to RX queue structure. 155 * @param[out] pkts 156 * Array to store received packets. 157 * @param pkts_n 158 * Maximum number of packets in array. 159 * 160 * @return 161 * Number of packets successfully received (<= pkts_n). 162 */ 163 static uint16_t 164 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 165 uint16_t pkts_n) 166 { 167 uint16_t n = 0; 168 unsigned int i; 169 #ifdef MLX5_PMD_SOFT_COUNTERS 170 uint32_t err_bytes = 0; 171 #endif 172 173 for (i = 0; i < pkts_n; ++i) { 174 struct rte_mbuf *pkt = pkts[i]; 175 176 if (pkt->packet_type == RTE_PTYPE_ALL_MASK) { 177 #ifdef MLX5_PMD_SOFT_COUNTERS 178 err_bytes += PKT_LEN(pkt); 179 #endif 180 rte_pktmbuf_free_seg(pkt); 181 } else { 182 pkts[n++] = pkt; 183 } 184 } 185 rxq->stats.idropped += (pkts_n - n); 186 #ifdef MLX5_PMD_SOFT_COUNTERS 187 /* Correct counters of errored completions. */ 188 rxq->stats.ipackets -= (pkts_n - n); 189 rxq->stats.ibytes -= err_bytes; 190 #endif 191 return n; 192 } 193 194 /** 195 * DPDK callback for vectorized RX. 196 * 197 * @param dpdk_rxq 198 * Generic pointer to RX queue structure. 199 * @param[out] pkts 200 * Array to store received packets. 201 * @param pkts_n 202 * Maximum number of packets in array. 203 * 204 * @return 205 * Number of packets successfully received (<= pkts_n). 206 */ 207 uint16_t 208 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 209 { 210 struct mlx5_rxq_data *rxq = dpdk_rxq; 211 uint16_t nb_rx; 212 uint64_t err = 0; 213 214 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); 215 if (unlikely(err)) 216 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 217 return nb_rx; 218 } 219 220 /** 221 * Check Tx queue flags are set for raw vectorized Tx. 222 * 223 * @param dev 224 * Pointer to Ethernet device. 225 * 226 * @return 227 * 1 if supported, negative errno value if not. 228 */ 229 int __attribute__((cold)) 230 mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev) 231 { 232 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 233 234 /* Doesn't support any offload. */ 235 if (offloads) 236 return -ENOTSUP; 237 return 1; 238 } 239 240 /** 241 * Check a device can support vectorized TX. 242 * 243 * @param dev 244 * Pointer to Ethernet device. 245 * 246 * @return 247 * 1 if supported, negative errno value if not. 248 */ 249 int __attribute__((cold)) 250 mlx5_check_vec_tx_support(struct rte_eth_dev *dev) 251 { 252 struct priv *priv = dev->data->dev_private; 253 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 254 255 if (!priv->config.tx_vec_en || 256 priv->txqs_n > MLX5_VPMD_MIN_TXQS || 257 priv->config.mps != MLX5_MPW_ENHANCED || 258 offloads & ~MLX5_VEC_TX_OFFLOAD_CAP) 259 return -ENOTSUP; 260 return 1; 261 } 262 263 /** 264 * Check a RX queue can support vectorized RX. 265 * 266 * @param rxq 267 * Pointer to RX queue. 268 * 269 * @return 270 * 1 if supported, negative errno value if not. 271 */ 272 int __attribute__((cold)) 273 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 274 { 275 struct mlx5_rxq_ctrl *ctrl = 276 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 277 278 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) 279 return -ENOTSUP; 280 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 281 return -ENOTSUP; 282 return 1; 283 } 284 285 /** 286 * Check a device can support vectorized RX. 287 * 288 * @param dev 289 * Pointer to Ethernet device. 290 * 291 * @return 292 * 1 if supported, negative errno value if not. 293 */ 294 int __attribute__((cold)) 295 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 296 { 297 struct priv *priv = dev->data->dev_private; 298 uint16_t i; 299 300 if (!priv->config.rx_vec_en) 301 return -ENOTSUP; 302 if (mlx5_mprq_enabled(dev)) 303 return -ENOTSUP; 304 /* All the configured queues should support. */ 305 for (i = 0; i < priv->rxqs_n; ++i) { 306 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 307 308 if (!rxq) 309 continue; 310 if (mlx5_rxq_check_vec_support(rxq) < 0) 311 break; 312 } 313 if (i != priv->rxqs_n) 314 return -ENOTSUP; 315 return 1; 316 } 317