1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <assert.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdlib.h> 10 11 /* Verbs header. */ 12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 13 #ifdef PEDANTIC 14 #pragma GCC diagnostic ignored "-Wpedantic" 15 #endif 16 #include <infiniband/verbs.h> 17 #include <infiniband/mlx5dv.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_mbuf.h> 23 #include <rte_mempool.h> 24 #include <rte_prefetch.h> 25 26 #include "mlx5.h" 27 #include "mlx5_utils.h" 28 #include "mlx5_rxtx.h" 29 #include "mlx5_rxtx_vec.h" 30 #include "mlx5_autoconf.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_prm.h" 33 34 #if defined RTE_ARCH_X86_64 35 #include "mlx5_rxtx_vec_sse.h" 36 #elif defined RTE_ARCH_ARM64 37 #include "mlx5_rxtx_vec_neon.h" 38 #else 39 #error "This should not be compiled if SIMD instructions are not supported." 40 #endif 41 42 /** 43 * Count the number of packets having same ol_flags and same metadata (if 44 * PKT_TX_METADATA is set in ol_flags), and calculate cs_flags. 45 * 46 * @param pkts 47 * Pointer to array of packets. 48 * @param pkts_n 49 * Number of packets. 50 * @param cs_flags 51 * Pointer of flags to be returned. 52 * @param metadata 53 * Pointer of metadata to be returned. 54 * @param txq_offloads 55 * Offloads enabled on Tx queue 56 * 57 * @return 58 * Number of packets having same ol_flags and metadata, if relevant. 59 */ 60 static inline unsigned int 61 txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags, 62 rte_be32_t *metadata, const uint64_t txq_offloads) 63 { 64 unsigned int pos; 65 const uint64_t cksum_ol_mask = 66 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | 67 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | 68 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; 69 rte_be32_t p0_metadata, pn_metadata; 70 71 if (!pkts_n) 72 return 0; 73 p0_metadata = pkts[0]->ol_flags & PKT_TX_METADATA ? 74 pkts[0]->tx_metadata : 0; 75 /* Count the number of packets having same offload parameters. */ 76 for (pos = 1; pos < pkts_n; ++pos) { 77 /* Check if packet has same checksum flags. */ 78 if ((txq_offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) && 79 ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & cksum_ol_mask)) 80 break; 81 /* Check if packet has same metadata. */ 82 if (txq_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) { 83 pn_metadata = pkts[pos]->ol_flags & PKT_TX_METADATA ? 84 pkts[pos]->tx_metadata : 0; 85 if (pn_metadata != p0_metadata) 86 break; 87 } 88 } 89 *cs_flags = txq_ol_cksum_to_cs(pkts[0]); 90 *metadata = p0_metadata; 91 return pos; 92 } 93 94 /** 95 * DPDK callback for vectorized TX. 96 * 97 * @param dpdk_txq 98 * Generic pointer to TX queue structure. 99 * @param[in] pkts 100 * Packets to transmit. 101 * @param pkts_n 102 * Number of packets in array. 103 * 104 * @return 105 * Number of packets successfully transmitted (<= pkts_n). 106 */ 107 uint16_t 108 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, 109 uint16_t pkts_n) 110 { 111 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 112 uint16_t nb_tx = 0; 113 114 while (pkts_n > nb_tx) { 115 uint16_t n; 116 uint16_t ret; 117 118 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 119 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0, 0); 120 nb_tx += ret; 121 if (!ret) 122 break; 123 } 124 return nb_tx; 125 } 126 127 /** 128 * DPDK callback for vectorized TX with multi-seg packets and offload. 129 * 130 * @param dpdk_txq 131 * Generic pointer to TX queue structure. 132 * @param[in] pkts 133 * Packets to transmit. 134 * @param pkts_n 135 * Number of packets in array. 136 * 137 * @return 138 * Number of packets successfully transmitted (<= pkts_n). 139 */ 140 uint16_t 141 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) 142 { 143 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 144 uint16_t nb_tx = 0; 145 146 while (pkts_n > nb_tx) { 147 uint8_t cs_flags = 0; 148 uint16_t n; 149 uint16_t ret; 150 rte_be32_t metadata = 0; 151 152 /* Transmit multi-seg packets in the head of pkts list. */ 153 if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) && 154 NB_SEGS(pkts[nb_tx]) > 1) 155 nb_tx += txq_scatter_v(txq, 156 &pkts[nb_tx], 157 pkts_n - nb_tx); 158 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 159 if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 160 n = txq_count_contig_single_seg(&pkts[nb_tx], n); 161 if (txq->offloads & (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | 162 DEV_TX_OFFLOAD_MATCH_METADATA)) 163 n = txq_calc_offload(&pkts[nb_tx], n, 164 &cs_flags, &metadata, 165 txq->offloads); 166 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags, metadata); 167 nb_tx += ret; 168 if (!ret) 169 break; 170 } 171 return nb_tx; 172 } 173 174 /** 175 * Skip error packets. 176 * 177 * @param rxq 178 * Pointer to RX queue structure. 179 * @param[out] pkts 180 * Array to store received packets. 181 * @param pkts_n 182 * Maximum number of packets in array. 183 * 184 * @return 185 * Number of packets successfully received (<= pkts_n). 186 */ 187 static uint16_t 188 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 189 uint16_t pkts_n) 190 { 191 uint16_t n = 0; 192 unsigned int i; 193 #ifdef MLX5_PMD_SOFT_COUNTERS 194 uint32_t err_bytes = 0; 195 #endif 196 197 for (i = 0; i < pkts_n; ++i) { 198 struct rte_mbuf *pkt = pkts[i]; 199 200 if (pkt->packet_type == RTE_PTYPE_ALL_MASK) { 201 #ifdef MLX5_PMD_SOFT_COUNTERS 202 err_bytes += PKT_LEN(pkt); 203 #endif 204 rte_pktmbuf_free_seg(pkt); 205 } else { 206 pkts[n++] = pkt; 207 } 208 } 209 rxq->stats.idropped += (pkts_n - n); 210 #ifdef MLX5_PMD_SOFT_COUNTERS 211 /* Correct counters of errored completions. */ 212 rxq->stats.ipackets -= (pkts_n - n); 213 rxq->stats.ibytes -= err_bytes; 214 #endif 215 return n; 216 } 217 218 /** 219 * DPDK callback for vectorized RX. 220 * 221 * @param dpdk_rxq 222 * Generic pointer to RX queue structure. 223 * @param[out] pkts 224 * Array to store received packets. 225 * @param pkts_n 226 * Maximum number of packets in array. 227 * 228 * @return 229 * Number of packets successfully received (<= pkts_n). 230 */ 231 uint16_t 232 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 233 { 234 struct mlx5_rxq_data *rxq = dpdk_rxq; 235 uint16_t nb_rx; 236 uint64_t err = 0; 237 238 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); 239 if (unlikely(err)) 240 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 241 return nb_rx; 242 } 243 244 /** 245 * Check Tx queue flags are set for raw vectorized Tx. 246 * 247 * @param dev 248 * Pointer to Ethernet device. 249 * 250 * @return 251 * 1 if supported, negative errno value if not. 252 */ 253 int __attribute__((cold)) 254 mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev) 255 { 256 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 257 258 /* Doesn't support any offload. */ 259 if (offloads) 260 return -ENOTSUP; 261 return 1; 262 } 263 264 /** 265 * Check a device can support vectorized TX. 266 * 267 * @param dev 268 * Pointer to Ethernet device. 269 * 270 * @return 271 * 1 if supported, negative errno value if not. 272 */ 273 int __attribute__((cold)) 274 mlx5_check_vec_tx_support(struct rte_eth_dev *dev) 275 { 276 struct priv *priv = dev->data->dev_private; 277 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 278 279 if (!priv->config.tx_vec_en || 280 priv->txqs_n > (unsigned int)priv->config.txqs_vec || 281 priv->config.mps != MLX5_MPW_ENHANCED || 282 offloads & ~MLX5_VEC_TX_OFFLOAD_CAP) 283 return -ENOTSUP; 284 return 1; 285 } 286 287 /** 288 * Check a RX queue can support vectorized RX. 289 * 290 * @param rxq 291 * Pointer to RX queue. 292 * 293 * @return 294 * 1 if supported, negative errno value if not. 295 */ 296 int __attribute__((cold)) 297 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 298 { 299 struct mlx5_rxq_ctrl *ctrl = 300 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 301 302 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) 303 return -ENOTSUP; 304 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 305 return -ENOTSUP; 306 return 1; 307 } 308 309 /** 310 * Check a device can support vectorized RX. 311 * 312 * @param dev 313 * Pointer to Ethernet device. 314 * 315 * @return 316 * 1 if supported, negative errno value if not. 317 */ 318 int __attribute__((cold)) 319 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 320 { 321 struct priv *priv = dev->data->dev_private; 322 uint16_t i; 323 324 if (!priv->config.rx_vec_en) 325 return -ENOTSUP; 326 if (mlx5_mprq_enabled(dev)) 327 return -ENOTSUP; 328 /* All the configured queues should support. */ 329 for (i = 0; i < priv->rxqs_n; ++i) { 330 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 331 332 if (!rxq) 333 continue; 334 if (mlx5_rxq_check_vec_support(rxq) < 0) 335 break; 336 } 337 if (i != priv->rxqs_n) 338 return -ENOTSUP; 339 return 1; 340 } 341