1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 #include <rte_mbuf.h> 11 #include <rte_mempool.h> 12 #include <rte_prefetch.h> 13 #include <rte_vect.h> 14 15 #include <mlx5_glue.h> 16 #include <mlx5_prm.h> 17 18 #include "mlx5_defs.h" 19 #include "mlx5.h" 20 #include "mlx5_utils.h" 21 #include "mlx5_rxtx.h" 22 #include "mlx5_rxtx_vec.h" 23 #include "mlx5_autoconf.h" 24 25 #if defined RTE_ARCH_X86_64 26 #include "mlx5_rxtx_vec_sse.h" 27 #elif defined RTE_ARCH_ARM64 28 #include "mlx5_rxtx_vec_neon.h" 29 #elif defined RTE_ARCH_PPC_64 30 #include "mlx5_rxtx_vec_altivec.h" 31 #else 32 #error "This should not be compiled if SIMD instructions are not supported." 33 #endif 34 35 /** 36 * Skip error packets. 37 * 38 * @param rxq 39 * Pointer to RX queue structure. 40 * @param[out] pkts 41 * Array to store received packets. 42 * @param pkts_n 43 * Maximum number of packets in array. 44 * 45 * @return 46 * Number of packets successfully received (<= pkts_n). 47 */ 48 static uint16_t 49 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 50 uint16_t pkts_n) 51 { 52 uint16_t n = 0; 53 unsigned int i; 54 #ifdef MLX5_PMD_SOFT_COUNTERS 55 uint32_t err_bytes = 0; 56 #endif 57 58 for (i = 0; i < pkts_n; ++i) { 59 struct rte_mbuf *pkt = pkts[i]; 60 61 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 62 #ifdef MLX5_PMD_SOFT_COUNTERS 63 err_bytes += PKT_LEN(pkt); 64 #endif 65 rte_pktmbuf_free_seg(pkt); 66 } else { 67 pkts[n++] = pkt; 68 } 69 } 70 rxq->stats.idropped += (pkts_n - n); 71 #ifdef MLX5_PMD_SOFT_COUNTERS 72 /* Correct counters of errored completions. */ 73 rxq->stats.ipackets -= (pkts_n - n); 74 rxq->stats.ibytes -= err_bytes; 75 #endif 76 mlx5_rx_err_handle(rxq, 1); 77 return n; 78 } 79 80 /** 81 * Replenish buffers for RX in bulk. 82 * 83 * @param rxq 84 * Pointer to RX queue structure. 85 */ 86 static inline void 87 mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq) 88 { 89 const uint16_t q_n = 1 << rxq->elts_n; 90 const uint16_t q_mask = q_n - 1; 91 uint16_t n = q_n - (rxq->rq_ci - rxq->rq_pi); 92 uint16_t elts_idx = rxq->rq_ci & q_mask; 93 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 94 volatile struct mlx5_wqe_data_seg *wq = 95 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx]; 96 unsigned int i; 97 98 if (n >= rxq->rq_repl_thresh) { 99 MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n)); 100 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) > 101 MLX5_VPMD_DESCS_PER_LOOP); 102 /* Not to cross queue end. */ 103 n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx); 104 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { 105 rxq->stats.rx_nombuf += n; 106 return; 107 } 108 for (i = 0; i < n; ++i) { 109 void *buf_addr; 110 111 /* 112 * In order to support the mbufs with external attached 113 * data buffer we should use the buf_addr pointer 114 * instead of rte_mbuf_buf_addr(). It touches the mbuf 115 * itself and may impact the performance. 116 */ 117 buf_addr = elts[i]->buf_addr; 118 wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr + 119 RTE_PKTMBUF_HEADROOM); 120 /* If there's a single MR, no need to replace LKey. */ 121 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) 122 > 1)) 123 wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]); 124 } 125 rxq->rq_ci += n; 126 /* Prevent overflowing into consumed mbufs. */ 127 elts_idx = rxq->rq_ci & q_mask; 128 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 129 (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf; 130 rte_io_wmb(); 131 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 132 } 133 } 134 135 /** 136 * Replenish buffers for MPRQ RX in bulk. 137 * 138 * @param rxq 139 * Pointer to RX queue structure. 140 */ 141 static inline void 142 mlx5_rx_mprq_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq) 143 { 144 const uint16_t wqe_n = 1 << rxq->elts_n; 145 const uint32_t strd_n = 1 << rxq->strd_num_n; 146 const uint32_t elts_n = wqe_n * strd_n; 147 const uint32_t wqe_mask = elts_n - 1; 148 uint32_t n = elts_n - (rxq->elts_ci - rxq->rq_pi); 149 uint32_t elts_idx = rxq->elts_ci & wqe_mask; 150 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 151 unsigned int i; 152 153 if (n >= rxq->rq_repl_thresh && 154 rxq->elts_ci - rxq->rq_pi <= rxq->rq_repl_thresh) { 155 MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n)); 156 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n) > 157 MLX5_VPMD_DESCS_PER_LOOP); 158 /* Not to cross queue end. */ 159 n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, elts_n - elts_idx); 160 /* Limit replenish number to threshold value. */ 161 n = RTE_MIN(n, rxq->rq_repl_thresh); 162 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { 163 rxq->stats.rx_nombuf += n; 164 return; 165 } 166 rxq->elts_ci += n; 167 /* Prevent overflowing into consumed mbufs. */ 168 elts_idx = rxq->elts_ci & wqe_mask; 169 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 170 (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf; 171 } 172 } 173 174 /** 175 * Copy or attach MPRQ buffers to RX SW ring. 176 * 177 * @param rxq 178 * Pointer to RX queue structure. 179 * @param pkts 180 * Pointer to array of packets to be stored. 181 * @param pkts_n 182 * Number of packets to be stored. 183 * 184 * @return 185 * Number of packets successfully copied/attached (<= pkts_n). 186 */ 187 static inline uint16_t 188 rxq_copy_mprq_mbuf_v(struct mlx5_rxq_data *rxq, 189 struct rte_mbuf **pkts, uint16_t pkts_n) 190 { 191 const uint16_t wqe_n = 1 << rxq->elts_n; 192 const uint16_t wqe_mask = wqe_n - 1; 193 const uint16_t strd_sz = 1 << rxq->strd_sz_n; 194 const uint32_t strd_n = 1 << rxq->strd_num_n; 195 const uint32_t elts_n = wqe_n * strd_n; 196 const uint32_t elts_mask = elts_n - 1; 197 uint32_t elts_idx = rxq->rq_pi & elts_mask; 198 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 199 uint32_t rq_ci = rxq->rq_ci; 200 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask]; 201 uint16_t copied = 0; 202 uint16_t i = 0; 203 204 for (i = 0; i < pkts_n; ++i) { 205 uint16_t strd_cnt; 206 enum mlx5_rqx_code rxq_code; 207 208 if (rxq->consumed_strd == strd_n) { 209 /* Replace WQE if the buffer is still in use. */ 210 mprq_buf_replace(rxq, rq_ci & wqe_mask); 211 /* Advance to the next WQE. */ 212 rxq->consumed_strd = 0; 213 rq_ci++; 214 buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask]; 215 } 216 217 if (!elts[i]->pkt_len) { 218 rxq->consumed_strd = strd_n; 219 rte_pktmbuf_free_seg(elts[i]); 220 #ifdef MLX5_PMD_SOFT_COUNTERS 221 rxq->stats.ipackets -= 1; 222 #endif 223 continue; 224 } 225 strd_cnt = (elts[i]->pkt_len / strd_sz) + 226 ((elts[i]->pkt_len % strd_sz) ? 1 : 0); 227 rxq_code = mprq_buf_to_pkt(rxq, elts[i], elts[i]->pkt_len, 228 buf, rxq->consumed_strd, strd_cnt); 229 rxq->consumed_strd += strd_cnt; 230 if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) { 231 rte_pktmbuf_free_seg(elts[i]); 232 #ifdef MLX5_PMD_SOFT_COUNTERS 233 rxq->stats.ipackets -= 1; 234 rxq->stats.ibytes -= elts[i]->pkt_len; 235 #endif 236 if (rxq_code == MLX5_RXQ_CODE_NOMBUF) { 237 ++rxq->stats.rx_nombuf; 238 break; 239 } 240 if (rxq_code == MLX5_RXQ_CODE_DROPPED) { 241 ++rxq->stats.idropped; 242 continue; 243 } 244 } 245 pkts[copied++] = elts[i]; 246 } 247 rxq->rq_pi += i; 248 rxq->cq_ci += i; 249 rte_io_wmb(); 250 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 251 if (rq_ci != rxq->rq_ci) { 252 rxq->rq_ci = rq_ci; 253 rte_io_wmb(); 254 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 255 } 256 return copied; 257 } 258 259 /** 260 * Receive burst of packets. An errored completion also consumes a mbuf, but the 261 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed 262 * before returning to application. 263 * 264 * @param rxq 265 * Pointer to RX queue structure. 266 * @param[out] pkts 267 * Array to store received packets. 268 * @param pkts_n 269 * Maximum number of packets in array. 270 * @param[out] err 271 * Pointer to a flag. Set non-zero value if pkts array has at least one error 272 * packet to handle. 273 * @param[out] no_cq 274 * Pointer to a boolean. Set true if no new CQE seen. 275 * 276 * @return 277 * Number of packets received including errors (<= pkts_n). 278 */ 279 static inline uint16_t 280 rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 281 uint16_t pkts_n, uint64_t *err, bool *no_cq) 282 { 283 const uint16_t q_n = 1 << rxq->cqe_n; 284 const uint16_t q_mask = q_n - 1; 285 const uint16_t e_n = 1 << rxq->elts_n; 286 const uint16_t e_mask = e_n - 1; 287 volatile struct mlx5_cqe *cq; 288 struct rte_mbuf **elts; 289 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; 290 uint16_t nocmp_n = 0; 291 uint16_t rcvd_pkt = 0; 292 unsigned int cq_idx = rxq->cq_ci & q_mask; 293 unsigned int elts_idx; 294 295 MLX5_ASSERT(rxq->sges_n == 0); 296 MLX5_ASSERT(rxq->cqe_n == rxq->elts_n); 297 cq = &(*rxq->cqes)[cq_idx]; 298 rte_prefetch0(cq); 299 rte_prefetch0(cq + 1); 300 rte_prefetch0(cq + 2); 301 rte_prefetch0(cq + 3); 302 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); 303 mlx5_rx_replenish_bulk_mbuf(rxq); 304 /* See if there're unreturned mbufs from compressed CQE. */ 305 rcvd_pkt = rxq->decompressed; 306 if (rcvd_pkt > 0) { 307 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); 308 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask], 309 pkts, rcvd_pkt); 310 rxq->rq_pi += rcvd_pkt; 311 rxq->decompressed -= rcvd_pkt; 312 pkts += rcvd_pkt; 313 } 314 elts_idx = rxq->rq_pi & e_mask; 315 elts = &(*rxq->elts)[elts_idx]; 316 /* Not to overflow pkts array. */ 317 pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP); 318 /* Not to cross queue end. */ 319 pkts_n = RTE_MIN(pkts_n, q_n - elts_idx); 320 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx); 321 if (!pkts_n) { 322 *no_cq = !rcvd_pkt; 323 return rcvd_pkt; 324 } 325 /* At this point, there shouldn't be any remaining packets. */ 326 MLX5_ASSERT(rxq->decompressed == 0); 327 /* Process all the CQEs */ 328 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); 329 /* If no new CQE seen, return without updating cq_db. */ 330 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) { 331 *no_cq = true; 332 return rcvd_pkt; 333 } 334 /* Update the consumer indexes for non-compressed CQEs. */ 335 MLX5_ASSERT(nocmp_n <= pkts_n); 336 rxq->cq_ci += nocmp_n; 337 rxq->rq_pi += nocmp_n; 338 rcvd_pkt += nocmp_n; 339 /* Decompress the last CQE if compressed. */ 340 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { 341 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); 342 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], 343 &elts[nocmp_n]); 344 rxq->cq_ci += rxq->decompressed; 345 /* Return more packets if needed. */ 346 if (nocmp_n < pkts_n) { 347 uint16_t n = rxq->decompressed; 348 349 n = RTE_MIN(n, pkts_n - nocmp_n); 350 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask], 351 &pkts[nocmp_n], n); 352 rxq->rq_pi += n; 353 rcvd_pkt += n; 354 rxq->decompressed -= n; 355 } 356 } 357 rte_io_wmb(); 358 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 359 *no_cq = !rcvd_pkt; 360 return rcvd_pkt; 361 } 362 363 /** 364 * DPDK callback for vectorized RX. 365 * 366 * @param dpdk_rxq 367 * Generic pointer to RX queue structure. 368 * @param[out] pkts 369 * Array to store received packets. 370 * @param pkts_n 371 * Maximum number of packets in array. 372 * 373 * @return 374 * Number of packets successfully received (<= pkts_n). 375 */ 376 uint16_t 377 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 378 { 379 struct mlx5_rxq_data *rxq = dpdk_rxq; 380 uint16_t nb_rx = 0; 381 uint16_t tn = 0; 382 uint64_t err = 0; 383 bool no_cq = false; 384 385 do { 386 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, 387 &err, &no_cq); 388 if (unlikely(err | rxq->err_state)) 389 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 390 tn += nb_rx; 391 if (unlikely(no_cq)) 392 break; 393 } while (tn != pkts_n); 394 return tn; 395 } 396 397 /** 398 * Receive burst of packets. An errored completion also consumes a mbuf, but the 399 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed 400 * before returning to application. 401 * 402 * @param rxq 403 * Pointer to RX queue structure. 404 * @param[out] pkts 405 * Array to store received packets. 406 * @param pkts_n 407 * Maximum number of packets in array. 408 * @param[out] err 409 * Pointer to a flag. Set non-zero value if pkts array has at least one error 410 * packet to handle. 411 * @param[out] no_cq 412 * Pointer to a boolean. Set true if no new CQE seen. 413 * 414 * @return 415 * Number of packets received including errors (<= pkts_n). 416 */ 417 static inline uint16_t 418 rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 419 uint16_t pkts_n, uint64_t *err, bool *no_cq) 420 { 421 const uint16_t q_n = 1 << rxq->cqe_n; 422 const uint16_t q_mask = q_n - 1; 423 const uint16_t wqe_n = 1 << rxq->elts_n; 424 const uint32_t strd_n = 1 << rxq->strd_num_n; 425 const uint32_t elts_n = wqe_n * strd_n; 426 const uint32_t elts_mask = elts_n - 1; 427 volatile struct mlx5_cqe *cq; 428 struct rte_mbuf **elts; 429 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; 430 uint16_t nocmp_n = 0; 431 uint16_t rcvd_pkt = 0; 432 uint16_t cp_pkt = 0; 433 unsigned int cq_idx = rxq->cq_ci & q_mask; 434 unsigned int elts_idx; 435 436 MLX5_ASSERT(rxq->sges_n == 0); 437 cq = &(*rxq->cqes)[cq_idx]; 438 rte_prefetch0(cq); 439 rte_prefetch0(cq + 1); 440 rte_prefetch0(cq + 2); 441 rte_prefetch0(cq + 3); 442 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); 443 mlx5_rx_mprq_replenish_bulk_mbuf(rxq); 444 /* See if there're unreturned mbufs from compressed CQE. */ 445 rcvd_pkt = rxq->decompressed; 446 if (rcvd_pkt > 0) { 447 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); 448 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, rcvd_pkt); 449 rxq->decompressed -= rcvd_pkt; 450 pkts += cp_pkt; 451 } 452 elts_idx = rxq->rq_pi & elts_mask; 453 elts = &(*rxq->elts)[elts_idx]; 454 /* Not to overflow pkts array. */ 455 pkts_n = RTE_ALIGN_FLOOR(pkts_n - cp_pkt, MLX5_VPMD_DESCS_PER_LOOP); 456 /* Not to cross queue end. */ 457 pkts_n = RTE_MIN(pkts_n, elts_n - elts_idx); 458 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx); 459 /* Not to move past the allocated mbufs. */ 460 pkts_n = RTE_MIN(pkts_n, rxq->elts_ci - rxq->rq_pi); 461 if (!pkts_n) { 462 *no_cq = !cp_pkt; 463 return cp_pkt; 464 } 465 /* At this point, there shouldn't be any remaining packets. */ 466 MLX5_ASSERT(rxq->decompressed == 0); 467 /* Process all the CQEs */ 468 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); 469 /* If no new CQE seen, return without updating cq_db. */ 470 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) { 471 *no_cq = true; 472 return cp_pkt; 473 } 474 /* Update the consumer indexes for non-compressed CQEs. */ 475 MLX5_ASSERT(nocmp_n <= pkts_n); 476 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, nocmp_n); 477 rcvd_pkt += cp_pkt; 478 /* Decompress the last CQE if compressed. */ 479 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { 480 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); 481 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], 482 &elts[nocmp_n]); 483 /* Return more packets if needed. */ 484 if (nocmp_n < pkts_n) { 485 uint16_t n = rxq->decompressed; 486 487 n = RTE_MIN(n, pkts_n - nocmp_n); 488 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, &pkts[cp_pkt], n); 489 rcvd_pkt += cp_pkt; 490 rxq->decompressed -= n; 491 } 492 } 493 *no_cq = !rcvd_pkt; 494 return rcvd_pkt; 495 } 496 497 /** 498 * DPDK callback for vectorized MPRQ RX. 499 * 500 * @param dpdk_rxq 501 * Generic pointer to RX queue structure. 502 * @param[out] pkts 503 * Array to store received packets. 504 * @param pkts_n 505 * Maximum number of packets in array. 506 * 507 * @return 508 * Number of packets successfully received (<= pkts_n). 509 */ 510 uint16_t 511 mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 512 { 513 struct mlx5_rxq_data *rxq = dpdk_rxq; 514 uint16_t nb_rx = 0; 515 uint16_t tn = 0; 516 uint64_t err = 0; 517 bool no_cq = false; 518 519 do { 520 nb_rx = rxq_burst_mprq_v(rxq, pkts + tn, pkts_n - tn, 521 &err, &no_cq); 522 if (unlikely(err | rxq->err_state)) 523 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 524 tn += nb_rx; 525 if (unlikely(no_cq)) 526 break; 527 } while (tn != pkts_n); 528 return tn; 529 } 530 531 /** 532 * Check a RX queue can support vectorized RX. 533 * 534 * @param rxq 535 * Pointer to RX queue. 536 * 537 * @return 538 * 1 if supported, negative errno value if not. 539 */ 540 int __rte_cold 541 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 542 { 543 struct mlx5_rxq_ctrl *ctrl = 544 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 545 546 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 547 return -ENOTSUP; 548 if (rxq->lro) 549 return -ENOTSUP; 550 return 1; 551 } 552 553 /** 554 * Check a device can support vectorized RX. 555 * 556 * @param dev 557 * Pointer to Ethernet device. 558 * 559 * @return 560 * 1 if supported, negative errno value if not. 561 */ 562 int __rte_cold 563 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 564 { 565 struct mlx5_priv *priv = dev->data->dev_private; 566 uint32_t i; 567 568 if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) 569 return -ENOTSUP; 570 if (!priv->config.rx_vec_en) 571 return -ENOTSUP; 572 /* All the configured queues should support. */ 573 for (i = 0; i < priv->rxqs_n; ++i) { 574 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 575 576 if (!rxq) 577 continue; 578 if (mlx5_rxq_check_vec_support(rxq) < 0) 579 break; 580 } 581 if (i != priv->rxqs_n) 582 return -ENOTSUP; 583 return 1; 584 } 585