1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <assert.h> 35 #include <stdint.h> 36 #include <string.h> 37 #include <stdlib.h> 38 39 /* Verbs header. */ 40 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 41 #ifdef PEDANTIC 42 #pragma GCC diagnostic ignored "-Wpedantic" 43 #endif 44 #include <infiniband/verbs.h> 45 #include <infiniband/mlx5dv.h> 46 #ifdef PEDANTIC 47 #pragma GCC diagnostic error "-Wpedantic" 48 #endif 49 50 #include <rte_mbuf.h> 51 #include <rte_mempool.h> 52 #include <rte_prefetch.h> 53 54 #include "mlx5.h" 55 #include "mlx5_utils.h" 56 #include "mlx5_rxtx.h" 57 #include "mlx5_rxtx_vec.h" 58 #include "mlx5_autoconf.h" 59 #include "mlx5_defs.h" 60 #include "mlx5_prm.h" 61 62 #if defined RTE_ARCH_X86_64 63 #include "mlx5_rxtx_vec_sse.h" 64 #elif defined RTE_ARCH_ARM64 65 #include "mlx5_rxtx_vec_neon.h" 66 #else 67 #error "This should not be compiled if SIMD instructions are not supported." 68 #endif 69 70 /** 71 * Count the number of continuous single segment packets. 72 * 73 * @param pkts 74 * Pointer to array of packets. 75 * @param pkts_n 76 * Number of packets. 77 * 78 * @return 79 * Number of continuous single segment packets. 80 */ 81 static inline unsigned int 82 txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n) 83 { 84 unsigned int pos; 85 86 if (!pkts_n) 87 return 0; 88 /* Count the number of continuous single segment packets. */ 89 for (pos = 0; pos < pkts_n; ++pos) 90 if (NB_SEGS(pkts[pos]) > 1) 91 break; 92 return pos; 93 } 94 95 /** 96 * Count the number of packets having same ol_flags and calculate cs_flags. 97 * 98 * @param txq 99 * Pointer to TX queue structure. 100 * @param pkts 101 * Pointer to array of packets. 102 * @param pkts_n 103 * Number of packets. 104 * @param cs_flags 105 * Pointer of flags to be returned. 106 * 107 * @return 108 * Number of packets having same ol_flags. 109 */ 110 static inline unsigned int 111 txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, 112 uint16_t pkts_n, uint8_t *cs_flags) 113 { 114 unsigned int pos; 115 const uint64_t ol_mask = 116 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | 117 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | 118 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; 119 120 if (!pkts_n) 121 return 0; 122 /* Count the number of packets having same ol_flags. */ 123 for (pos = 1; pos < pkts_n; ++pos) 124 if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) 125 break; 126 /* Should open another MPW session for the rest. */ 127 if (pkts[0]->ol_flags & 128 (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { 129 const uint64_t is_tunneled = 130 pkts[0]->ol_flags & 131 (PKT_TX_TUNNEL_GRE | 132 PKT_TX_TUNNEL_VXLAN); 133 134 if (is_tunneled && txq->tunnel_en) { 135 *cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | 136 MLX5_ETH_WQE_L4_INNER_CSUM; 137 if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM) 138 *cs_flags |= MLX5_ETH_WQE_L3_CSUM; 139 } else { 140 *cs_flags = MLX5_ETH_WQE_L3_CSUM | 141 MLX5_ETH_WQE_L4_CSUM; 142 } 143 } 144 return pos; 145 } 146 147 /** 148 * DPDK callback for vectorized TX. 149 * 150 * @param dpdk_txq 151 * Generic pointer to TX queue structure. 152 * @param[in] pkts 153 * Packets to transmit. 154 * @param pkts_n 155 * Number of packets in array. 156 * 157 * @return 158 * Number of packets successfully transmitted (<= pkts_n). 159 */ 160 uint16_t 161 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, 162 uint16_t pkts_n) 163 { 164 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 165 uint16_t nb_tx = 0; 166 167 while (pkts_n > nb_tx) { 168 uint16_t n; 169 uint16_t ret; 170 171 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 172 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0); 173 nb_tx += ret; 174 if (!ret) 175 break; 176 } 177 return nb_tx; 178 } 179 180 /** 181 * DPDK callback for vectorized TX with multi-seg packets and offload. 182 * 183 * @param dpdk_txq 184 * Generic pointer to TX queue structure. 185 * @param[in] pkts 186 * Packets to transmit. 187 * @param pkts_n 188 * Number of packets in array. 189 * 190 * @return 191 * Number of packets successfully transmitted (<= pkts_n). 192 */ 193 uint16_t 194 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) 195 { 196 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 197 uint16_t nb_tx = 0; 198 199 while (pkts_n > nb_tx) { 200 uint8_t cs_flags = 0; 201 uint16_t n; 202 uint16_t ret; 203 204 /* Transmit multi-seg packets in the head of pkts list. */ 205 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) && 206 NB_SEGS(pkts[nb_tx]) > 1) 207 nb_tx += txq_scatter_v(txq, 208 &pkts[nb_tx], 209 pkts_n - nb_tx); 210 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 211 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS)) 212 n = txq_check_multiseg(&pkts[nb_tx], n); 213 if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) 214 n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags); 215 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); 216 nb_tx += ret; 217 if (!ret) 218 break; 219 } 220 return nb_tx; 221 } 222 223 /** 224 * Skip error packets. 225 * 226 * @param rxq 227 * Pointer to RX queue structure. 228 * @param[out] pkts 229 * Array to store received packets. 230 * @param pkts_n 231 * Maximum number of packets in array. 232 * 233 * @return 234 * Number of packets successfully received (<= pkts_n). 235 */ 236 static uint16_t 237 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 238 uint16_t pkts_n) 239 { 240 uint16_t n = 0; 241 unsigned int i; 242 #ifdef MLX5_PMD_SOFT_COUNTERS 243 uint32_t err_bytes = 0; 244 #endif 245 246 for (i = 0; i < pkts_n; ++i) { 247 struct rte_mbuf *pkt = pkts[i]; 248 249 if (pkt->packet_type == RTE_PTYPE_ALL_MASK) { 250 #ifdef MLX5_PMD_SOFT_COUNTERS 251 err_bytes += PKT_LEN(pkt); 252 #endif 253 rte_pktmbuf_free_seg(pkt); 254 } else { 255 pkts[n++] = pkt; 256 } 257 } 258 rxq->stats.idropped += (pkts_n - n); 259 #ifdef MLX5_PMD_SOFT_COUNTERS 260 /* Correct counters of errored completions. */ 261 rxq->stats.ipackets -= (pkts_n - n); 262 rxq->stats.ibytes -= err_bytes; 263 #endif 264 rxq->pending_err = 0; 265 return n; 266 } 267 268 /** 269 * DPDK callback for vectorized RX. 270 * 271 * @param dpdk_rxq 272 * Generic pointer to RX queue structure. 273 * @param[out] pkts 274 * Array to store received packets. 275 * @param pkts_n 276 * Maximum number of packets in array. 277 * 278 * @return 279 * Number of packets successfully received (<= pkts_n). 280 */ 281 uint16_t 282 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 283 { 284 struct mlx5_rxq_data *rxq = dpdk_rxq; 285 uint16_t nb_rx; 286 287 nb_rx = rxq_burst_v(rxq, pkts, pkts_n); 288 if (unlikely(rxq->pending_err)) 289 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 290 return nb_rx; 291 } 292 293 /** 294 * Check Tx queue flags are set for raw vectorized Tx. 295 * 296 * @param priv 297 * Pointer to private structure. 298 * 299 * @return 300 * 1 if supported, negative errno value if not. 301 */ 302 int __attribute__((cold)) 303 priv_check_raw_vec_tx_support(struct priv *priv) 304 { 305 uint16_t i; 306 307 /* All the configured queues should support. */ 308 for (i = 0; i < priv->txqs_n; ++i) { 309 struct mlx5_txq_data *txq = (*priv->txqs)[i]; 310 311 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) || 312 !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) 313 break; 314 } 315 if (i != priv->txqs_n) 316 return -ENOTSUP; 317 return 1; 318 } 319 320 /** 321 * Check a device can support vectorized TX. 322 * 323 * @param priv 324 * Pointer to private structure. 325 * 326 * @return 327 * 1 if supported, negative errno value if not. 328 */ 329 int __attribute__((cold)) 330 priv_check_vec_tx_support(struct priv *priv) 331 { 332 if (!priv->tx_vec_en || 333 priv->txqs_n > MLX5_VPMD_MIN_TXQS || 334 priv->mps != MLX5_MPW_ENHANCED || 335 priv->tso) 336 return -ENOTSUP; 337 return 1; 338 } 339 340 /** 341 * Check a RX queue can support vectorized RX. 342 * 343 * @param rxq 344 * Pointer to RX queue. 345 * 346 * @return 347 * 1 if supported, negative errno value if not. 348 */ 349 int __attribute__((cold)) 350 rxq_check_vec_support(struct mlx5_rxq_data *rxq) 351 { 352 struct mlx5_rxq_ctrl *ctrl = 353 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 354 355 if (!ctrl->priv->rx_vec_en || rxq->sges_n != 0) 356 return -ENOTSUP; 357 return 1; 358 } 359 360 /** 361 * Check a device can support vectorized RX. 362 * 363 * @param priv 364 * Pointer to private structure. 365 * 366 * @return 367 * 1 if supported, negative errno value if not. 368 */ 369 int __attribute__((cold)) 370 priv_check_vec_rx_support(struct priv *priv) 371 { 372 uint16_t i; 373 374 if (!priv->rx_vec_en) 375 return -ENOTSUP; 376 /* All the configured queues should support. */ 377 for (i = 0; i < priv->rxqs_n; ++i) { 378 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 379 380 if (!rxq) 381 continue; 382 if (rxq_check_vec_support(rxq) < 0) 383 break; 384 } 385 if (i != priv->rxqs_n) 386 return -ENOTSUP; 387 return 1; 388 } 389