1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <assert.h> 35 #include <stdint.h> 36 #include <string.h> 37 #include <stdlib.h> 38 39 /* Verbs header. */ 40 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 41 #ifdef PEDANTIC 42 #pragma GCC diagnostic ignored "-Wpedantic" 43 #endif 44 #include <infiniband/verbs.h> 45 #include <infiniband/mlx5dv.h> 46 #ifdef PEDANTIC 47 #pragma GCC diagnostic error "-Wpedantic" 48 #endif 49 50 #include <rte_mbuf.h> 51 #include <rte_mempool.h> 52 #include <rte_prefetch.h> 53 54 #include "mlx5.h" 55 #include "mlx5_utils.h" 56 #include "mlx5_rxtx.h" 57 #include "mlx5_rxtx_vec.h" 58 #include "mlx5_autoconf.h" 59 #include "mlx5_defs.h" 60 #include "mlx5_prm.h" 61 62 #if defined RTE_ARCH_X86_64 63 #include "mlx5_rxtx_vec_sse.h" 64 #elif defined RTE_ARCH_ARM64 65 #include "mlx5_rxtx_vec_neon.h" 66 #else 67 #error "This should not be compiled if SIMD instructions are not supported." 68 #endif 69 70 /** 71 * Count the number of packets having same ol_flags and calculate cs_flags. 72 * 73 * @param txq 74 * Pointer to TX queue structure. 75 * @param pkts 76 * Pointer to array of packets. 77 * @param pkts_n 78 * Number of packets. 79 * @param cs_flags 80 * Pointer of flags to be returned. 81 * 82 * @return 83 * Number of packets having same ol_flags. 84 */ 85 static inline unsigned int 86 txq_calc_offload(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, 87 uint16_t pkts_n, uint8_t *cs_flags) 88 { 89 unsigned int pos; 90 const uint64_t ol_mask = 91 PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | 92 PKT_TX_UDP_CKSUM | PKT_TX_TUNNEL_GRE | 93 PKT_TX_TUNNEL_VXLAN | PKT_TX_OUTER_IP_CKSUM; 94 95 if (!pkts_n) 96 return 0; 97 /* Count the number of packets having same ol_flags. */ 98 for (pos = 1; pos < pkts_n; ++pos) 99 if ((pkts[pos]->ol_flags ^ pkts[0]->ol_flags) & ol_mask) 100 break; 101 *cs_flags = txq_ol_cksum_to_cs(txq, pkts[0]); 102 return pos; 103 } 104 105 /** 106 * DPDK callback for vectorized TX. 107 * 108 * @param dpdk_txq 109 * Generic pointer to TX queue structure. 110 * @param[in] pkts 111 * Packets to transmit. 112 * @param pkts_n 113 * Number of packets in array. 114 * 115 * @return 116 * Number of packets successfully transmitted (<= pkts_n). 117 */ 118 uint16_t 119 mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, 120 uint16_t pkts_n) 121 { 122 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 123 uint16_t nb_tx = 0; 124 125 while (pkts_n > nb_tx) { 126 uint16_t n; 127 uint16_t ret; 128 129 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 130 ret = txq_burst_v(txq, &pkts[nb_tx], n, 0); 131 nb_tx += ret; 132 if (!ret) 133 break; 134 } 135 return nb_tx; 136 } 137 138 /** 139 * DPDK callback for vectorized TX with multi-seg packets and offload. 140 * 141 * @param dpdk_txq 142 * Generic pointer to TX queue structure. 143 * @param[in] pkts 144 * Packets to transmit. 145 * @param pkts_n 146 * Number of packets in array. 147 * 148 * @return 149 * Number of packets successfully transmitted (<= pkts_n). 150 */ 151 uint16_t 152 mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) 153 { 154 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 155 uint16_t nb_tx = 0; 156 157 while (pkts_n > nb_tx) { 158 uint8_t cs_flags = 0; 159 uint16_t n; 160 uint16_t ret; 161 162 /* Transmit multi-seg packets in the head of pkts list. */ 163 if ((txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) && 164 NB_SEGS(pkts[nb_tx]) > 1) 165 nb_tx += txq_scatter_v(txq, 166 &pkts[nb_tx], 167 pkts_n - nb_tx); 168 n = RTE_MIN((uint16_t)(pkts_n - nb_tx), MLX5_VPMD_TX_MAX_BURST); 169 if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 170 n = txq_count_contig_single_seg(&pkts[nb_tx], n); 171 if (txq->offloads & MLX5_VEC_TX_CKSUM_OFFLOAD_CAP) 172 n = txq_calc_offload(txq, &pkts[nb_tx], n, &cs_flags); 173 ret = txq_burst_v(txq, &pkts[nb_tx], n, cs_flags); 174 nb_tx += ret; 175 if (!ret) 176 break; 177 } 178 return nb_tx; 179 } 180 181 /** 182 * Skip error packets. 183 * 184 * @param rxq 185 * Pointer to RX queue structure. 186 * @param[out] pkts 187 * Array to store received packets. 188 * @param pkts_n 189 * Maximum number of packets in array. 190 * 191 * @return 192 * Number of packets successfully received (<= pkts_n). 193 */ 194 static uint16_t 195 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 196 uint16_t pkts_n) 197 { 198 uint16_t n = 0; 199 unsigned int i; 200 #ifdef MLX5_PMD_SOFT_COUNTERS 201 uint32_t err_bytes = 0; 202 #endif 203 204 for (i = 0; i < pkts_n; ++i) { 205 struct rte_mbuf *pkt = pkts[i]; 206 207 if (pkt->packet_type == RTE_PTYPE_ALL_MASK) { 208 #ifdef MLX5_PMD_SOFT_COUNTERS 209 err_bytes += PKT_LEN(pkt); 210 #endif 211 rte_pktmbuf_free_seg(pkt); 212 } else { 213 pkts[n++] = pkt; 214 } 215 } 216 rxq->stats.idropped += (pkts_n - n); 217 #ifdef MLX5_PMD_SOFT_COUNTERS 218 /* Correct counters of errored completions. */ 219 rxq->stats.ipackets -= (pkts_n - n); 220 rxq->stats.ibytes -= err_bytes; 221 #endif 222 return n; 223 } 224 225 /** 226 * DPDK callback for vectorized RX. 227 * 228 * @param dpdk_rxq 229 * Generic pointer to RX queue structure. 230 * @param[out] pkts 231 * Array to store received packets. 232 * @param pkts_n 233 * Maximum number of packets in array. 234 * 235 * @return 236 * Number of packets successfully received (<= pkts_n). 237 */ 238 uint16_t 239 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 240 { 241 struct mlx5_rxq_data *rxq = dpdk_rxq; 242 uint16_t nb_rx; 243 uint64_t err = 0; 244 245 nb_rx = rxq_burst_v(rxq, pkts, pkts_n, &err); 246 if (unlikely(err)) 247 nb_rx = rxq_handle_pending_error(rxq, pkts, nb_rx); 248 return nb_rx; 249 } 250 251 /** 252 * Check Tx queue flags are set for raw vectorized Tx. 253 * 254 * @param priv 255 * Pointer to private structure. 256 * @param dev 257 * Pointer to rte_eth_dev structure. 258 * 259 * @return 260 * 1 if supported, negative errno value if not. 261 */ 262 int __attribute__((cold)) 263 priv_check_raw_vec_tx_support(__rte_unused struct priv *priv, 264 struct rte_eth_dev *dev) 265 { 266 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 267 268 /* Doesn't support any offload. */ 269 if (offloads) 270 return -ENOTSUP; 271 return 1; 272 } 273 274 /** 275 * Check a device can support vectorized TX. 276 * 277 * @param priv 278 * Pointer to private structure. 279 * @param dev 280 * Pointer to rte_eth_dev structure. 281 * 282 * @return 283 * 1 if supported, negative errno value if not. 284 */ 285 int __attribute__((cold)) 286 priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev) 287 { 288 uint64_t offloads = dev->data->dev_conf.txmode.offloads; 289 290 if (!priv->config.tx_vec_en || 291 priv->txqs_n > MLX5_VPMD_MIN_TXQS || 292 priv->config.mps != MLX5_MPW_ENHANCED || 293 offloads & ~MLX5_VEC_TX_OFFLOAD_CAP) 294 return -ENOTSUP; 295 return 1; 296 } 297 298 /** 299 * Check a RX queue can support vectorized RX. 300 * 301 * @param rxq 302 * Pointer to RX queue. 303 * 304 * @return 305 * 1 if supported, negative errno value if not. 306 */ 307 int __attribute__((cold)) 308 rxq_check_vec_support(struct mlx5_rxq_data *rxq) 309 { 310 struct mlx5_rxq_ctrl *ctrl = 311 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 312 313 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 314 return -ENOTSUP; 315 return 1; 316 } 317 318 /** 319 * Check a device can support vectorized RX. 320 * 321 * @param priv 322 * Pointer to private structure. 323 * 324 * @return 325 * 1 if supported, negative errno value if not. 326 */ 327 int __attribute__((cold)) 328 priv_check_vec_rx_support(struct priv *priv) 329 { 330 uint16_t i; 331 332 if (!priv->config.rx_vec_en) 333 return -ENOTSUP; 334 /* All the configured queues should support. */ 335 for (i = 0; i < priv->rxqs_n; ++i) { 336 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 337 338 if (!rxq) 339 continue; 340 if (rxq_check_vec_support(rxq) < 0) 341 break; 342 } 343 if (i != priv->rxqs_n) 344 return -ENOTSUP; 345 return 1; 346 } 347