1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <assert.h> 35 #include <stdint.h> 36 #include <string.h> 37 #include <stdlib.h> 38 39 /* Verbs header. */ 40 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 41 #ifdef PEDANTIC 42 #pragma GCC diagnostic ignored "-Wpedantic" 43 #endif 44 #include <infiniband/verbs.h> 45 #include <infiniband/mlx5dv.h> 46 #ifdef PEDANTIC 47 #pragma GCC diagnostic error "-Wpedantic" 48 #endif 49 50 #include <rte_mbuf.h> 51 #include <rte_mempool.h> 52 #include <rte_prefetch.h> 53 54 #include "mlx5.h" 55 #include "mlx5_utils.h" 56 #include "mlx5_rxtx.h" 57 #include "mlx5_rxtx_vec.h" 58 #include "mlx5_autoconf.h" 59 #include "mlx5_defs.h" 60 #include "mlx5_prm.h" 61 62 #ifdef RTE_ARCH_X86_64 63 #include "mlx5_rxtx_vec_sse.h" 64 #else 65 #error "This should not be compiled if SIMD instructions are not supported." 66 #endif 67 68 /** 69 * Count the number of continuous single segment packets. 70 * 71 * @param pkts 72 * Pointer to array of packets. 73 * @param pkts_n 74 * Number of packets. 75 * 76 * @return 77 * Number of continuous single segment packets. 78 */ 79 static inline unsigned int 80 txq_check_multiseg(struct rte_mbuf **pkts, uint16_t pkts_n) 81 { 82 unsigned int pos; 83 84 if (!pkts_n) 85 return 0; 86 /* Count the number of continuous single segment packets. */ 87 for (pos = 0; pos < pkts_n; ++pos) 88 if (NB_SEGS(pkts[pos]) > 1) 89 break; 90 return pos; 91 } 92 93 /** 94 * Count the number of packets having same ol_flags and calculate cs_flags. 95 * 96 * @param txq 97 * Pointer to TX queue structure. 98 * @param pkts 99 * Pointer to array of packets. 100 * @param pkts_n 101 * Number of packets. 102 * @param cs_flags 103 * Pointer of flags to be returned. 104 * 105 * @return 106 * Number of packets having same ol_flags. 107 */ 108 static inline unsigned int 109 txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, 110 uint16_t pkts_n, uint8_t *cs_flags) 111 { 112 unsigned int pos; 113 const uint64_t ol_mask = 114 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | 115 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | 116 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; 117 118 if (!pkts_n) 119 return 0; 120 /* Count the number of packets having same ol_flags. */ 121 for (pos = 1; pos < pkts_n; ++pos) 122 if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) 123 break; 124 /* Should open another MPW session for the rest. */ 125 if (pkts[0]->ol_flags & 126 (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { 127 const uint64_t is_tunneled = 128 pkts[0]->ol_flags & 129 (PKT_TX_TUNNEL_GRE | 130 PKT_TX_TUNNEL_VXLAN); 131 132 if (is_tunneled && txq->tunnel_en) { 133 *cs_flags = MLX5_ETH_WQE_L3_INNER_CSUM | 134 MLX5_ETH_WQE_L4_INNER_CSUM; 135 if (pkts[0]->ol_flags & PKT_TX_OUTER_IP_CKSUM) 136 *cs_flags |= MLX5_ETH_WQE_L3_CSUM; 137 } else { 138 *cs_flags = MLX5_ETH_WQE_L3_CSUM | 139 MLX5_ETH_WQE_L4_CSUM; 140 } 141 } 142 return pos; 143 } 144 145 /** 146 * DPDK callback for vectorized TX. 147 * 148 * @param dpdk_txq 149 * Generic pointer to TX queue structure. 150 * @param[in] pkts 151 * Packets to transmit. 152 * @param pkts_n 153 * Number of packets in array. 154 * 155 * @return 156 * Number of packets successfully transmitted (<= pkts_n). 157 */ 158 uint16_t 159 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, 160 uint16_t pkts_n) 161 { 162 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 163 uint16_t nb_tx = 0; 164 165 while (pkts_n > nb_tx) { 166 uint16_t n; 167 uint16_t ret; 168 169 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 170 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0); 171 nb_tx += ret; 172 if (!ret) 173 break; 174 } 175 return nb_tx; 176 } 177 178 /** 179 * DPDK callback for vectorized TX with multi-seg packets and offload. 180 * 181 * @param dpdk_txq 182 * Generic pointer to TX queue structure. 183 * @param[in] pkts 184 * Packets to transmit. 185 * @param pkts_n 186 * Number of packets in array. 187 * 188 * @return 189 * Number of packets successfully transmitted (<= pkts_n). 190 */ 191 uint16_t 192 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) 193 { 194 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 195 uint16_t nb_tx = 0; 196 197 while (pkts_n > nb_tx) { 198 uint8_t cs_flags = 0; 199 uint16_t n; 200 uint16_t ret; 201 202 /* Transmit multi-seg packets in the head of pkts list. */ 203 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) && 204 NB_SEGS(pkts[nb_tx]) > 1) 205 nb_tx += txq_scatter_v(txq, 206 &pkts[nb_tx], 207 pkts_n - nb_tx); 208 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 209 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS)) 210 n = txq_check_multiseg(&pkts[nb_tx], n); 211 if (!(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) 212 n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags); 213 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); 214 nb_tx += ret; 215 if (!ret) 216 break; 217 } 218 return nb_tx; 219 } 220 221 /** 222 * Skip error packets. 223 * 224 * @param rxq 225 * Pointer to RX queue structure. 226 * @param[out] pkts 227 * Array to store received packets. 228 * @param pkts_n 229 * Maximum number of packets in array. 230 * 231 * @return 232 * Number of packets successfully received (<= pkts_n). 233 */ 234 static uint16_t 235 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 236 uint16_t pkts_n) 237 { 238 uint16_t n = 0; 239 unsigned int i; 240 #ifdef MLX5_PMD_SOFT_COUNTERS 241 uint32_t err_bytes = 0; 242 #endif 243 244 for (i = 0; i < pkts_n; ++i) { 245 struct rte_mbuf *pkt = pkts[i]; 246 247 if (pkt->packet_type == RTE_PTYPE_ALL_MASK) { 248 #ifdef MLX5_PMD_SOFT_COUNTERS 249 err_bytes += PKT_LEN(pkt); 250 #endif 251 rte_pktmbuf_free_seg(pkt); 252 } else { 253 pkts[n++] = pkt; 254 } 255 } 256 rxq->stats.idropped += (pkts_n - n); 257 #ifdef MLX5_PMD_SOFT_COUNTERS 258 /* Correct counters of errored completions. */ 259 rxq->stats.ipackets -= (pkts_n - n); 260 rxq->stats.ibytes -= err_bytes; 261 #endif 262 rxq->pending_err = 0; 263 return n; 264 } 265 266 /** 267 * DPDK callback for vectorized RX. 268 * 269 * @param dpdk_rxq 270 * Generic pointer to RX queue structure. 271 * @param[out] pkts 272 * Array to store received packets. 273 * @param pkts_n 274 * Maximum number of packets in array. 275 * 276 * @return 277 * Number of packets successfully received (<= pkts_n). 278 */ 279 uint16_t 280 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 281 { 282 struct mlx5_rxq_data *rxq = dpdk_rxq; 283 uint16_t nb_rx; 284 285 nb_rx = rxq_burst_v(rxq, pkts, pkts_n); 286 if (unlikely(rxq->pending_err)) 287 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 288 return nb_rx; 289 } 290 291 /** 292 * Check Tx queue flags are set for raw vectorized Tx. 293 * 294 * @param priv 295 * Pointer to private structure. 296 * 297 * @return 298 * 1 if supported, negative errno value if not. 299 */ 300 int __attribute__((cold)) 301 priv_check_raw_vec_tx_support(struct priv *priv) 302 { 303 uint16_t i; 304 305 /* All the configured queues should support. */ 306 for (i = 0; i < priv->txqs_n; ++i) { 307 struct mlx5_txq_data *txq = (*priv->txqs)[i]; 308 309 if (!(txq->flags & ETH_TXQ_FLAGS_NOMULTSEGS) || 310 !(txq->flags & ETH_TXQ_FLAGS_NOOFFLOADS)) 311 break; 312 } 313 if (i != priv->txqs_n) 314 return -ENOTSUP; 315 return 1; 316 } 317 318 /** 319 * Check a device can support vectorized TX. 320 * 321 * @param priv 322 * Pointer to private structure. 323 * 324 * @return 325 * 1 if supported, negative errno value if not. 326 */ 327 int __attribute__((cold)) 328 priv_check_vec_tx_support(struct priv *priv) 329 { 330 if (!priv->tx_vec_en || 331 priv->txqs_n > MLX5_VPMD_MIN_TXQS || 332 priv->mps != MLX5_MPW_ENHANCED || 333 priv->tso) 334 return -ENOTSUP; 335 return 1; 336 } 337 338 /** 339 * Check a RX queue can support vectorized RX. 340 * 341 * @param rxq 342 * Pointer to RX queue. 343 * 344 * @return 345 * 1 if supported, negative errno value if not. 346 */ 347 int __attribute__((cold)) 348 rxq_check_vec_support(struct mlx5_rxq_data *rxq) 349 { 350 struct mlx5_rxq_ctrl *ctrl = 351 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 352 353 if (!ctrl->priv->rx_vec_en || rxq->sges_n != 0) 354 return -ENOTSUP; 355 return 1; 356 } 357 358 /** 359 * Check a device can support vectorized RX. 360 * 361 * @param priv 362 * Pointer to private structure. 363 * 364 * @return 365 * 1 if supported, negative errno value if not. 366 */ 367 int __attribute__((cold)) 368 priv_check_vec_rx_support(struct priv *priv) 369 { 370 uint16_t i; 371 372 if (!priv->rx_vec_en) 373 return -ENOTSUP; 374 /* All the configured queues should support. */ 375 for (i = 0; i < priv->rxqs_n; ++i) { 376 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 377 378 if (!rxq) 379 continue; 380 if (rxq_check_vec_support(rxq) < 0) 381 break; 382 } 383 if (i != priv->rxqs_n) 384 return -ENOTSUP; 385 return 1; 386 } 387