1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <assert.h> 7 #include <stdint.h> 8 #include <string.h> 9 #include <stdlib.h> 10 11 /* Verbs header. */ 12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 13 #ifdef PEDANTIC 14 #pragma GCC diagnostic ignored "-Wpedantic" 15 #endif 16 #include <infiniband/verbs.h> 17 #include <infiniband/mlx5dv.h> 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic error "-Wpedantic" 20 #endif 21 22 #include <rte_mbuf.h> 23 #include <rte_mempool.h> 24 #include <rte_prefetch.h> 25 26 #include "mlx5.h" 27 #include "mlx5_utils.h" 28 #include "mlx5_rxtx.h" 29 #include "mlx5_rxtx_vec.h" 30 #include "mlx5_autoconf.h" 31 #include "mlx5_defs.h" 32 #include "mlx5_prm.h" 33 34 #if defined RTE_ARCH_X86_64 35 #include "mlx5_rxtx_vec_sse.h" 36 #elif defined RTE_ARCH_ARM64 37 #include "mlx5_rxtx_vec_neon.h" 38 #else 39 #error "This should not be compiled if SIMD instructions are not supported." 40 #endif 41 42 /** 43 * Count the number of packets having same ol_flags and same metadata (if 44 * PKT_TX_METADATA is set in ol_flags), and calculate cs_flags. 45 * 46 * @param pkts 47 * Pointer to array of packets. 48 * @param pkts_n 49 * Number of packets. 50 * @param cs_flags 51 * Pointer of flags to be returned. 52 * @param metadata 53 * Pointer of metadata to be returned. 54 * @param txq_offloads 55 * Offloads enabled on Tx queue 56 * 57 * @return 58 * Number of packets having same ol_flags and metadata, if relevant. 59 */ 60 static inline unsigned int 61 txq_calc_offload(struct rte_mbuf **pkts, uint16_t pkts_n, uint8_t *cs_flags, 62 rte_be32_t *metadata, const uint64_t txq_offloads) 63 { 64 unsigned int pos; 65 const uint64_t cksum_ol_mask = 66 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | 67 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | 68 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; 69 rte_be32_t p0_metadata, pn_metadata; 70 71 if (!pkts_n) 72 return 0; 73 p0_metadata = pkts[0]->ol_flags & PKT_TX_METADATA ? 74 pkts[0]->tx_metadata : 0; 75 /* Count the number of packets having same offload parameters. */ 76 for (pos = 1; pos < pkts_n; ++pos) { 77 /* Check if packet has same checksum flags. */ 78 if ((txq_offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) && 79 ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & cksum_ol_mask)) 80 break; 81 /* Check if packet has same metadata. */ 82 if (txq_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) { 83 pn_metadata = pkts[pos]->ol_flags & PKT_TX_METADATA ? 84 pkts[pos]->tx_metadata : 0; 85 if (pn_metadata != p0_metadata) 86 break; 87 } 88 } 89 *cs_flags = txq_ol_cksum_to_cs(pkts[0]); 90 *metadata = p0_metadata; 91 return pos; 92 } 93 94 /** 95 * DPDK callback for vectorized TX. 96 * 97 * @param dpdk_txq 98 * Generic pointer to TX queue structure. 99 * @param[in] pkts 100 * Packets to transmit. 101 * @param pkts_n 102 * Number of packets in array. 103 * 104 * @return 105 * Number of packets successfully transmitted (<= pkts_n). 106 */ 107 uint16_t 108 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, 109 uint16_t pkts_n) 110 { 111 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 112 uint16_t nb_tx = 0; 113 114 while (pkts_n > nb_tx) { 115 uint16_t n; 116 uint16_t ret; 117 118 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 119 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0, 0); 120 nb_tx += ret; 121 if (!ret) 122 break; 123 } 124 return nb_tx; 125 } 126 127 /** 128 * DPDK callback for vectorized TX with multi-seg packets and offload. 129 * 130 * @param dpdk_txq 131 * Generic pointer to TX queue structure. 132 * @param[in] pkts 133 * Packets to transmit. 134 * @param pkts_n 135 * Number of packets in array. 136 * 137 * @return 138 * Number of packets successfully transmitted (<= pkts_n). 139 */ 140 uint16_t 141 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) 142 { 143 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 144 uint16_t nb_tx = 0; 145 146 while (pkts_n > nb_tx) { 147 uint8_t cs_flags = 0; 148 uint16_t n; 149 uint16_t ret; 150 rte_be32_t metadata = 0; 151 152 /* Transmit multi-seg packets in the head of pkts list. */ 153 if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) && 154 NB_SEGS(pkts[nb_tx]) > 1) 155 nb_tx += txq_scatter_v(txq, 156 &pkts[nb_tx], 157 pkts_n - nb_tx); 158 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 159 if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 160 n = txq_count_contig_single_seg(&pkts[nb_tx], n); 161 if (txq->offloads & (MLX5_VEC_TX_CKSUM_OFFLOAD_CAP | 162 DEV_TX_OFFLOAD_MATCH_METADATA)) 163 n = txq_calc_offload(&pkts[nb_tx], n, 164 &cs_flags, &metadata, 165 txq->offloads); 166 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags, metadata); 167 nb_tx += ret; 168 if (!ret) 169 break; 170 } 171 return nb_tx; 172 } 173 174 /** 175 * Skip error packets. 176 * 177 * @param rxq 178 * Pointer to RX queue structure. 179 * @param[out] pkts 180 * Array to store received packets. 181 * @param pkts_n 182 * Maximum number of packets in array. 183 * 184 * @return 185 * Number of packets successfully received (<= pkts_n). 186 */ 187 static uint16_t 188 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 189 uint16_t pkts_n) 190 { 191 uint16_t n = 0; 192 unsigned int i; 193 #ifdef MLX5_PMD_SOFT_COUNTERS 194 uint32_t err_bytes = 0; 195 #endif 196 197 for (i = 0; i < pkts_n; ++i) { 198 struct rte_mbuf *pkt = pkts[i]; 199 200 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 201 #ifdef MLX5_PMD_SOFT_COUNTERS 202 err_bytes += PKT_LEN(pkt); 203 #endif 204 rte_pktmbuf_free_seg(pkt); 205 } else { 206 pkts[n++] = pkt; 207 } 208 } 209 rxq->stats.idropped += (pkts_n - n); 210 #ifdef MLX5_PMD_SOFT_COUNTERS 211 /* Correct counters of errored completions. */ 212 rxq->stats.ipackets -= (pkts_n - n); 213 rxq->stats.ibytes -= err_bytes; 214 #endif 215 mlx5_rx_err_handle(rxq, 1); 216 return n; 217 } 218 219 /** 220 * DPDK callback for vectorized RX. 221 * 222 * @param dpdk_rxq 223 * Generic pointer to RX queue structure. 224 * @param[out] pkts 225 * Array to store received packets. 226 * @param pkts_n 227 * Maximum number of packets in array. 228 * 229 * @return 230 * Number of packets successfully received (<= pkts_n). 231 */ 232 uint16_t 233 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 234 { 235 struct mlx5_rxq_data *rxq = dpdk_rxq; 236 uint16_t nb_rx; 237 uint64_t err = 0; 238 239 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); 240 if (unlikely(err | rxq->err_state)) 241 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 242 return nb_rx; 243 } 244 245 /** 246 * Check Tx queue flags are set for raw vectorized Tx. 247 * 248 * @param dev 249 * Pointer to Ethernet device. 250 * 251 * @return 252 * 1 if supported, negative errno value if not. 253 */ 254 int __attribute__((cold)) 255 mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev) 256 { 257 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 258 259 /* Doesn't support any offload. */ 260 if (offloads) 261 return -ENOTSUP; 262 return 1; 263 } 264 265 /** 266 * Check a device can support vectorized TX. 267 * 268 * @param dev 269 * Pointer to Ethernet device. 270 * 271 * @return 272 * 1 if supported, negative errno value if not. 273 */ 274 int __attribute__((cold)) 275 mlx5_check_vec_tx_support(struct rte_eth_dev *dev) 276 { 277 struct mlx5_priv *priv = dev->data->dev_private; 278 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 279 280 if (!priv->config.tx_vec_en || 281 priv->txqs_n > (unsigned int)priv->config.txqs_vec || 282 priv->config.mps != MLX5_MPW_ENHANCED || 283 offloads & ~MLX5_VEC_TX_OFFLOAD_CAP) 284 return -ENOTSUP; 285 return 1; 286 } 287 288 /** 289 * Check a RX queue can support vectorized RX. 290 * 291 * @param rxq 292 * Pointer to RX queue. 293 * 294 * @return 295 * 1 if supported, negative errno value if not. 296 */ 297 int __attribute__((cold)) 298 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 299 { 300 struct mlx5_rxq_ctrl *ctrl = 301 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 302 303 if (mlx5_mprq_enabled(ETH_DEV(ctrl->priv))) 304 return -ENOTSUP; 305 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 306 return -ENOTSUP; 307 return 1; 308 } 309 310 /** 311 * Check a device can support vectorized RX. 312 * 313 * @param dev 314 * Pointer to Ethernet device. 315 * 316 * @return 317 * 1 if supported, negative errno value if not. 318 */ 319 int __attribute__((cold)) 320 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 321 { 322 struct mlx5_priv *priv = dev->data->dev_private; 323 uint16_t i; 324 325 if (!priv->config.rx_vec_en) 326 return -ENOTSUP; 327 if (mlx5_mprq_enabled(dev)) 328 return -ENOTSUP; 329 /* All the configured queues should support. */ 330 for (i = 0; i < priv->rxqs_n; ++i) { 331 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 332 333 if (!rxq) 334 continue; 335 if (mlx5_rxq_check_vec_support(rxq) < 0) 336 break; 337 } 338 if (i != priv->rxqs_n) 339 return -ENOTSUP; 340 return 1; 341 } 342