1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 #include <rte_mbuf.h> 11 #include <rte_mempool.h> 12 #include <rte_prefetch.h> 13 #include <rte_vect.h> 14 15 #include <mlx5_glue.h> 16 #include <mlx5_prm.h> 17 18 #include "mlx5_defs.h" 19 #include "mlx5.h" 20 #include "mlx5_utils.h" 21 #include "mlx5_rxtx.h" 22 #include "mlx5_rx.h" 23 #include "mlx5_rxtx_vec.h" 24 #include "mlx5_autoconf.h" 25 26 #if defined RTE_ARCH_X86_64 27 #include "mlx5_rxtx_vec_sse.h" 28 #elif defined RTE_ARCH_ARM64 29 #include "mlx5_rxtx_vec_neon.h" 30 #elif defined RTE_ARCH_PPC_64 31 #include "mlx5_rxtx_vec_altivec.h" 32 #else 33 #error "This should not be compiled if SIMD instructions are not supported." 34 #endif 35 36 /** 37 * Skip error packets. 38 * 39 * @param rxq 40 * Pointer to RX queue structure. 41 * @param[out] pkts 42 * Array to store received packets. 43 * @param pkts_n 44 * Maximum number of packets in array. 45 * 46 * @return 47 * Number of packets successfully received (<= pkts_n). 48 */ 49 static uint16_t 50 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 51 uint16_t pkts_n) 52 { 53 uint16_t n = 0; 54 unsigned int i; 55 #ifdef MLX5_PMD_SOFT_COUNTERS 56 uint32_t err_bytes = 0; 57 #endif 58 59 for (i = 0; i < pkts_n; ++i) { 60 struct rte_mbuf *pkt = pkts[i]; 61 62 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 63 #ifdef MLX5_PMD_SOFT_COUNTERS 64 err_bytes += PKT_LEN(pkt); 65 #endif 66 rte_pktmbuf_free_seg(pkt); 67 } else { 68 pkts[n++] = pkt; 69 } 70 } 71 rxq->stats.idropped += (pkts_n - n); 72 #ifdef MLX5_PMD_SOFT_COUNTERS 73 /* Correct counters of errored completions. */ 74 rxq->stats.ipackets -= (pkts_n - n); 75 rxq->stats.ibytes -= err_bytes; 76 #endif 77 mlx5_rx_err_handle(rxq, 1); 78 return n; 79 } 80 81 /** 82 * Replenish buffers for RX in bulk. 83 * 84 * @param rxq 85 * Pointer to RX queue structure. 86 */ 87 static inline void 88 mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq) 89 { 90 const uint16_t q_n = 1 << rxq->elts_n; 91 const uint16_t q_mask = q_n - 1; 92 uint16_t n = q_n - (rxq->rq_ci - rxq->rq_pi); 93 uint16_t elts_idx = rxq->rq_ci & q_mask; 94 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 95 volatile struct mlx5_wqe_data_seg *wq = 96 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx]; 97 unsigned int i; 98 99 if (n >= rxq->rq_repl_thresh) { 100 MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n)); 101 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) > 102 MLX5_VPMD_DESCS_PER_LOOP); 103 /* Not to cross queue end. */ 104 n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx); 105 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { 106 rxq->stats.rx_nombuf += n; 107 return; 108 } 109 for (i = 0; i < n; ++i) { 110 void *buf_addr; 111 112 /* 113 * In order to support the mbufs with external attached 114 * data buffer we should use the buf_addr pointer 115 * instead of rte_mbuf_buf_addr(). It touches the mbuf 116 * itself and may impact the performance. 117 */ 118 buf_addr = elts[i]->buf_addr; 119 wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr + 120 RTE_PKTMBUF_HEADROOM); 121 /* If there's a single MR, no need to replace LKey. */ 122 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) 123 > 1)) 124 wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]); 125 } 126 rxq->rq_ci += n; 127 /* Prevent overflowing into consumed mbufs. */ 128 elts_idx = rxq->rq_ci & q_mask; 129 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 130 (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf; 131 rte_io_wmb(); 132 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 133 } 134 } 135 136 /** 137 * Replenish buffers for MPRQ RX in bulk. 138 * 139 * @param rxq 140 * Pointer to RX queue structure. 141 */ 142 static inline void 143 mlx5_rx_mprq_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq) 144 { 145 const uint16_t wqe_n = 1 << rxq->elts_n; 146 const uint32_t strd_n = 1 << rxq->strd_num_n; 147 const uint32_t elts_n = wqe_n * strd_n; 148 const uint32_t wqe_mask = elts_n - 1; 149 uint32_t n = elts_n - (rxq->elts_ci - rxq->rq_pi); 150 uint32_t elts_idx = rxq->elts_ci & wqe_mask; 151 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 152 unsigned int i; 153 154 if (n >= rxq->rq_repl_thresh && 155 rxq->elts_ci - rxq->rq_pi <= rxq->rq_repl_thresh) { 156 MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n)); 157 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n) > 158 MLX5_VPMD_DESCS_PER_LOOP); 159 /* Not to cross queue end. */ 160 n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, elts_n - elts_idx); 161 /* Limit replenish number to threshold value. */ 162 n = RTE_MIN(n, rxq->rq_repl_thresh); 163 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { 164 rxq->stats.rx_nombuf += n; 165 return; 166 } 167 rxq->elts_ci += n; 168 /* Prevent overflowing into consumed mbufs. */ 169 elts_idx = rxq->elts_ci & wqe_mask; 170 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 171 (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf; 172 } 173 } 174 175 /** 176 * Copy or attach MPRQ buffers to RX SW ring. 177 * 178 * @param rxq 179 * Pointer to RX queue structure. 180 * @param pkts 181 * Pointer to array of packets to be stored. 182 * @param pkts_n 183 * Number of packets to be stored. 184 * 185 * @return 186 * Number of packets successfully copied/attached (<= pkts_n). 187 */ 188 static inline uint16_t 189 rxq_copy_mprq_mbuf_v(struct mlx5_rxq_data *rxq, 190 struct rte_mbuf **pkts, uint16_t pkts_n) 191 { 192 const uint16_t wqe_n = 1 << rxq->elts_n; 193 const uint16_t wqe_mask = wqe_n - 1; 194 const uint16_t strd_sz = 1 << rxq->strd_sz_n; 195 const uint32_t strd_n = 1 << rxq->strd_num_n; 196 const uint32_t elts_n = wqe_n * strd_n; 197 const uint32_t elts_mask = elts_n - 1; 198 uint32_t elts_idx = rxq->rq_pi & elts_mask; 199 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 200 uint32_t rq_ci = rxq->rq_ci; 201 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask]; 202 uint16_t copied = 0; 203 uint16_t i = 0; 204 205 for (i = 0; i < pkts_n; ++i) { 206 uint16_t strd_cnt; 207 enum mlx5_rqx_code rxq_code; 208 209 if (rxq->consumed_strd == strd_n) { 210 /* Replace WQE if the buffer is still in use. */ 211 mprq_buf_replace(rxq, rq_ci & wqe_mask); 212 /* Advance to the next WQE. */ 213 rxq->consumed_strd = 0; 214 rq_ci++; 215 buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask]; 216 } 217 218 if (!elts[i]->pkt_len) { 219 rxq->consumed_strd = strd_n; 220 rte_pktmbuf_free_seg(elts[i]); 221 #ifdef MLX5_PMD_SOFT_COUNTERS 222 rxq->stats.ipackets -= 1; 223 #endif 224 continue; 225 } 226 strd_cnt = (elts[i]->pkt_len / strd_sz) + 227 ((elts[i]->pkt_len % strd_sz) ? 1 : 0); 228 rxq_code = mprq_buf_to_pkt(rxq, elts[i], elts[i]->pkt_len, 229 buf, rxq->consumed_strd, strd_cnt); 230 rxq->consumed_strd += strd_cnt; 231 if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) { 232 rte_pktmbuf_free_seg(elts[i]); 233 #ifdef MLX5_PMD_SOFT_COUNTERS 234 rxq->stats.ipackets -= 1; 235 rxq->stats.ibytes -= elts[i]->pkt_len; 236 #endif 237 if (rxq_code == MLX5_RXQ_CODE_NOMBUF) { 238 ++rxq->stats.rx_nombuf; 239 break; 240 } 241 if (rxq_code == MLX5_RXQ_CODE_DROPPED) { 242 ++rxq->stats.idropped; 243 continue; 244 } 245 } 246 pkts[copied++] = elts[i]; 247 } 248 rxq->rq_pi += i; 249 rxq->cq_ci += i; 250 rte_io_wmb(); 251 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 252 if (rq_ci != rxq->rq_ci) { 253 rxq->rq_ci = rq_ci; 254 rte_io_wmb(); 255 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 256 } 257 return copied; 258 } 259 260 /** 261 * Receive burst of packets. An errored completion also consumes a mbuf, but the 262 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed 263 * before returning to application. 264 * 265 * @param rxq 266 * Pointer to RX queue structure. 267 * @param[out] pkts 268 * Array to store received packets. 269 * @param pkts_n 270 * Maximum number of packets in array. 271 * @param[out] err 272 * Pointer to a flag. Set non-zero value if pkts array has at least one error 273 * packet to handle. 274 * @param[out] no_cq 275 * Pointer to a boolean. Set true if no new CQE seen. 276 * 277 * @return 278 * Number of packets received including errors (<= pkts_n). 279 */ 280 static inline uint16_t 281 rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 282 uint16_t pkts_n, uint64_t *err, bool *no_cq) 283 { 284 const uint16_t q_n = 1 << rxq->cqe_n; 285 const uint16_t q_mask = q_n - 1; 286 const uint16_t e_n = 1 << rxq->elts_n; 287 const uint16_t e_mask = e_n - 1; 288 volatile struct mlx5_cqe *cq; 289 struct rte_mbuf **elts; 290 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; 291 uint16_t nocmp_n = 0; 292 uint16_t rcvd_pkt = 0; 293 unsigned int cq_idx = rxq->cq_ci & q_mask; 294 unsigned int elts_idx; 295 296 MLX5_ASSERT(rxq->sges_n == 0); 297 MLX5_ASSERT(rxq->cqe_n == rxq->elts_n); 298 cq = &(*rxq->cqes)[cq_idx]; 299 rte_prefetch0(cq); 300 rte_prefetch0(cq + 1); 301 rte_prefetch0(cq + 2); 302 rte_prefetch0(cq + 3); 303 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); 304 mlx5_rx_replenish_bulk_mbuf(rxq); 305 /* See if there're unreturned mbufs from compressed CQE. */ 306 rcvd_pkt = rxq->decompressed; 307 if (rcvd_pkt > 0) { 308 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); 309 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask], 310 pkts, rcvd_pkt); 311 rxq->rq_pi += rcvd_pkt; 312 rxq->decompressed -= rcvd_pkt; 313 pkts += rcvd_pkt; 314 } 315 elts_idx = rxq->rq_pi & e_mask; 316 elts = &(*rxq->elts)[elts_idx]; 317 /* Not to overflow pkts array. */ 318 pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP); 319 /* Not to cross queue end. */ 320 pkts_n = RTE_MIN(pkts_n, q_n - elts_idx); 321 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx); 322 if (!pkts_n) { 323 *no_cq = !rcvd_pkt; 324 return rcvd_pkt; 325 } 326 /* At this point, there shouldn't be any remaining packets. */ 327 MLX5_ASSERT(rxq->decompressed == 0); 328 /* Process all the CQEs */ 329 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); 330 /* If no new CQE seen, return without updating cq_db. */ 331 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) { 332 *no_cq = true; 333 return rcvd_pkt; 334 } 335 /* Update the consumer indexes for non-compressed CQEs. */ 336 MLX5_ASSERT(nocmp_n <= pkts_n); 337 rxq->cq_ci += nocmp_n; 338 rxq->rq_pi += nocmp_n; 339 rcvd_pkt += nocmp_n; 340 /* Decompress the last CQE if compressed. */ 341 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { 342 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); 343 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], 344 &elts[nocmp_n]); 345 rxq->cq_ci += rxq->decompressed; 346 /* Return more packets if needed. */ 347 if (nocmp_n < pkts_n) { 348 uint16_t n = rxq->decompressed; 349 350 n = RTE_MIN(n, pkts_n - nocmp_n); 351 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask], 352 &pkts[nocmp_n], n); 353 rxq->rq_pi += n; 354 rcvd_pkt += n; 355 rxq->decompressed -= n; 356 } 357 } 358 rte_io_wmb(); 359 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 360 *no_cq = !rcvd_pkt; 361 return rcvd_pkt; 362 } 363 364 /** 365 * DPDK callback for vectorized RX. 366 * 367 * @param dpdk_rxq 368 * Generic pointer to RX queue structure. 369 * @param[out] pkts 370 * Array to store received packets. 371 * @param pkts_n 372 * Maximum number of packets in array. 373 * 374 * @return 375 * Number of packets successfully received (<= pkts_n). 376 */ 377 uint16_t 378 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 379 { 380 struct mlx5_rxq_data *rxq = dpdk_rxq; 381 uint16_t nb_rx = 0; 382 uint16_t tn = 0; 383 uint64_t err = 0; 384 bool no_cq = false; 385 386 do { 387 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, 388 &err, &no_cq); 389 if (unlikely(err | rxq->err_state)) 390 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 391 tn += nb_rx; 392 if (unlikely(no_cq)) 393 break; 394 } while (tn != pkts_n); 395 return tn; 396 } 397 398 /** 399 * Receive burst of packets. An errored completion also consumes a mbuf, but the 400 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed 401 * before returning to application. 402 * 403 * @param rxq 404 * Pointer to RX queue structure. 405 * @param[out] pkts 406 * Array to store received packets. 407 * @param pkts_n 408 * Maximum number of packets in array. 409 * @param[out] err 410 * Pointer to a flag. Set non-zero value if pkts array has at least one error 411 * packet to handle. 412 * @param[out] no_cq 413 * Pointer to a boolean. Set true if no new CQE seen. 414 * 415 * @return 416 * Number of packets received including errors (<= pkts_n). 417 */ 418 static inline uint16_t 419 rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 420 uint16_t pkts_n, uint64_t *err, bool *no_cq) 421 { 422 const uint16_t q_n = 1 << rxq->cqe_n; 423 const uint16_t q_mask = q_n - 1; 424 const uint16_t wqe_n = 1 << rxq->elts_n; 425 const uint32_t strd_n = 1 << rxq->strd_num_n; 426 const uint32_t elts_n = wqe_n * strd_n; 427 const uint32_t elts_mask = elts_n - 1; 428 volatile struct mlx5_cqe *cq; 429 struct rte_mbuf **elts; 430 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; 431 uint16_t nocmp_n = 0; 432 uint16_t rcvd_pkt = 0; 433 uint16_t cp_pkt = 0; 434 unsigned int cq_idx = rxq->cq_ci & q_mask; 435 unsigned int elts_idx; 436 437 MLX5_ASSERT(rxq->sges_n == 0); 438 cq = &(*rxq->cqes)[cq_idx]; 439 rte_prefetch0(cq); 440 rte_prefetch0(cq + 1); 441 rte_prefetch0(cq + 2); 442 rte_prefetch0(cq + 3); 443 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); 444 mlx5_rx_mprq_replenish_bulk_mbuf(rxq); 445 /* See if there're unreturned mbufs from compressed CQE. */ 446 rcvd_pkt = rxq->decompressed; 447 if (rcvd_pkt > 0) { 448 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); 449 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, rcvd_pkt); 450 rxq->decompressed -= rcvd_pkt; 451 pkts += cp_pkt; 452 } 453 elts_idx = rxq->rq_pi & elts_mask; 454 elts = &(*rxq->elts)[elts_idx]; 455 /* Not to overflow pkts array. */ 456 pkts_n = RTE_ALIGN_FLOOR(pkts_n - cp_pkt, MLX5_VPMD_DESCS_PER_LOOP); 457 /* Not to cross queue end. */ 458 pkts_n = RTE_MIN(pkts_n, elts_n - elts_idx); 459 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx); 460 /* Not to move past the allocated mbufs. */ 461 pkts_n = RTE_MIN(pkts_n, rxq->elts_ci - rxq->rq_pi); 462 if (!pkts_n) { 463 *no_cq = !cp_pkt; 464 return cp_pkt; 465 } 466 /* At this point, there shouldn't be any remaining packets. */ 467 MLX5_ASSERT(rxq->decompressed == 0); 468 /* Process all the CQEs */ 469 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); 470 /* If no new CQE seen, return without updating cq_db. */ 471 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) { 472 *no_cq = true; 473 return cp_pkt; 474 } 475 /* Update the consumer indexes for non-compressed CQEs. */ 476 MLX5_ASSERT(nocmp_n <= pkts_n); 477 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, nocmp_n); 478 rcvd_pkt += cp_pkt; 479 /* Decompress the last CQE if compressed. */ 480 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { 481 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); 482 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], 483 &elts[nocmp_n]); 484 /* Return more packets if needed. */ 485 if (nocmp_n < pkts_n) { 486 uint16_t n = rxq->decompressed; 487 488 n = RTE_MIN(n, pkts_n - nocmp_n); 489 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, &pkts[cp_pkt], n); 490 rcvd_pkt += cp_pkt; 491 rxq->decompressed -= n; 492 } 493 } 494 *no_cq = !rcvd_pkt; 495 return rcvd_pkt; 496 } 497 498 /** 499 * DPDK callback for vectorized MPRQ RX. 500 * 501 * @param dpdk_rxq 502 * Generic pointer to RX queue structure. 503 * @param[out] pkts 504 * Array to store received packets. 505 * @param pkts_n 506 * Maximum number of packets in array. 507 * 508 * @return 509 * Number of packets successfully received (<= pkts_n). 510 */ 511 uint16_t 512 mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 513 { 514 struct mlx5_rxq_data *rxq = dpdk_rxq; 515 uint16_t nb_rx = 0; 516 uint16_t tn = 0; 517 uint64_t err = 0; 518 bool no_cq = false; 519 520 do { 521 nb_rx = rxq_burst_mprq_v(rxq, pkts + tn, pkts_n - tn, 522 &err, &no_cq); 523 if (unlikely(err | rxq->err_state)) 524 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 525 tn += nb_rx; 526 if (unlikely(no_cq)) 527 break; 528 } while (tn != pkts_n); 529 return tn; 530 } 531 532 /** 533 * Check a RX queue can support vectorized RX. 534 * 535 * @param rxq 536 * Pointer to RX queue. 537 * 538 * @return 539 * 1 if supported, negative errno value if not. 540 */ 541 int __rte_cold 542 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 543 { 544 struct mlx5_rxq_ctrl *ctrl = 545 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 546 547 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 548 return -ENOTSUP; 549 if (rxq->lro) 550 return -ENOTSUP; 551 return 1; 552 } 553 554 /** 555 * Check a device can support vectorized RX. 556 * 557 * @param dev 558 * Pointer to Ethernet device. 559 * 560 * @return 561 * 1 if supported, negative errno value if not. 562 */ 563 int __rte_cold 564 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 565 { 566 struct mlx5_priv *priv = dev->data->dev_private; 567 uint32_t i; 568 569 if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) 570 return -ENOTSUP; 571 if (!priv->config.rx_vec_en) 572 return -ENOTSUP; 573 /* All the configured queues should support. */ 574 for (i = 0; i < priv->rxqs_n; ++i) { 575 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 576 577 if (!rxq) 578 continue; 579 if (mlx5_rxq_check_vec_support(rxq) < 0) 580 break; 581 } 582 if (i != priv->rxqs_n) 583 return -ENOTSUP; 584 return 1; 585 } 586