1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 #include <rte_mbuf.h> 11 #include <rte_mempool.h> 12 #include <rte_prefetch.h> 13 #include <rte_vect.h> 14 15 #include <mlx5_glue.h> 16 #include <mlx5_prm.h> 17 18 #include "mlx5_defs.h" 19 #include "mlx5.h" 20 #include "mlx5_utils.h" 21 #include "mlx5_rxtx.h" 22 #include "mlx5_rxtx_vec.h" 23 #include "mlx5_autoconf.h" 24 25 #if defined RTE_ARCH_X86_64 26 #include "mlx5_rxtx_vec_sse.h" 27 #elif defined RTE_ARCH_ARM64 28 #include "mlx5_rxtx_vec_neon.h" 29 #elif defined RTE_ARCH_PPC_64 30 #include "mlx5_rxtx_vec_altivec.h" 31 #else 32 #error "This should not be compiled if SIMD instructions are not supported." 33 #endif 34 35 /** 36 * Skip error packets. 37 * 38 * @param rxq 39 * Pointer to RX queue structure. 40 * @param[out] pkts 41 * Array to store received packets. 42 * @param pkts_n 43 * Maximum number of packets in array. 44 * 45 * @return 46 * Number of packets successfully received (<= pkts_n). 47 */ 48 static uint16_t 49 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 50 uint16_t pkts_n) 51 { 52 uint16_t n = 0; 53 unsigned int i; 54 #ifdef MLX5_PMD_SOFT_COUNTERS 55 uint32_t err_bytes = 0; 56 #endif 57 58 for (i = 0; i < pkts_n; ++i) { 59 struct rte_mbuf *pkt = pkts[i]; 60 61 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) { 62 #ifdef MLX5_PMD_SOFT_COUNTERS 63 err_bytes += PKT_LEN(pkt); 64 #endif 65 rte_pktmbuf_free_seg(pkt); 66 } else { 67 pkts[n++] = pkt; 68 } 69 } 70 rxq->stats.idropped += (pkts_n - n); 71 #ifdef MLX5_PMD_SOFT_COUNTERS 72 /* Correct counters of errored completions. */ 73 rxq->stats.ipackets -= (pkts_n - n); 74 rxq->stats.ibytes -= err_bytes; 75 #endif 76 mlx5_rx_err_handle(rxq, 1); 77 return n; 78 } 79 80 /** 81 * Replenish buffers for RX in bulk. 82 * 83 * @param rxq 84 * Pointer to RX queue structure. 85 */ 86 static inline void 87 mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq) 88 { 89 const uint16_t q_n = 1 << rxq->elts_n; 90 const uint16_t q_mask = q_n - 1; 91 uint16_t n = q_n - (rxq->rq_ci - rxq->rq_pi); 92 uint16_t elts_idx = rxq->rq_ci & q_mask; 93 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 94 volatile struct mlx5_wqe_data_seg *wq = 95 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx]; 96 unsigned int i; 97 98 if (n >= rxq->rq_repl_thresh) { 99 MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n)); 100 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) > 101 MLX5_VPMD_DESCS_PER_LOOP); 102 /* Not to cross queue end. */ 103 n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx); 104 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { 105 rxq->stats.rx_nombuf += n; 106 return; 107 } 108 for (i = 0; i < n; ++i) { 109 void *buf_addr; 110 111 /* 112 * In order to support the mbufs with external attached 113 * data buffer we should use the buf_addr pointer 114 * instead of rte_mbuf_buf_addr(). It touches the mbuf 115 * itself and may impact the performance. 116 */ 117 buf_addr = elts[i]->buf_addr; 118 wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr + 119 RTE_PKTMBUF_HEADROOM); 120 /* If there's a single MR, no need to replace LKey. */ 121 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) 122 > 1)) 123 wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]); 124 } 125 rxq->rq_ci += n; 126 /* Prevent overflowing into consumed mbufs. */ 127 elts_idx = rxq->rq_ci & q_mask; 128 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 129 (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf; 130 rte_io_wmb(); 131 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 132 } 133 } 134 135 /** 136 * Replenish buffers for MPRQ RX in bulk. 137 * 138 * @param rxq 139 * Pointer to RX queue structure. 140 */ 141 static inline void 142 mlx5_rx_mprq_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq) 143 { 144 const uint16_t wqe_n = 1 << rxq->elts_n; 145 const uint32_t strd_n = 1 << rxq->strd_num_n; 146 const uint32_t elts_n = wqe_n * strd_n; 147 const uint32_t wqe_mask = elts_n - 1; 148 uint32_t n = rxq->elts_ci - rxq->rq_pi; 149 uint32_t elts_idx = rxq->elts_ci & wqe_mask; 150 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 151 152 if (n <= rxq->rq_repl_thresh) { 153 MLX5_ASSERT(n + MLX5_VPMD_RX_MAX_BURST >= 154 MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n)); 155 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n) > 156 MLX5_VPMD_DESCS_PER_LOOP); 157 /* Not to cross queue end. */ 158 n = RTE_MIN(n + MLX5_VPMD_RX_MAX_BURST, elts_n - elts_idx); 159 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { 160 rxq->stats.rx_nombuf += n; 161 return; 162 } 163 rxq->elts_ci += n; 164 } 165 } 166 167 /** 168 * Copy or attach MPRQ buffers to RX SW ring. 169 * 170 * @param rxq 171 * Pointer to RX queue structure. 172 * @param pkts 173 * Pointer to array of packets to be stored. 174 * @param pkts_n 175 * Number of packets to be stored. 176 * 177 * @return 178 * Number of packets successfully copied/attached (<= pkts_n). 179 */ 180 static inline uint16_t 181 rxq_copy_mprq_mbuf_v(struct mlx5_rxq_data *rxq, 182 struct rte_mbuf **pkts, uint16_t pkts_n) 183 { 184 const uint16_t wqe_n = 1 << rxq->elts_n; 185 const uint16_t wqe_mask = wqe_n - 1; 186 const uint16_t strd_sz = 1 << rxq->strd_sz_n; 187 const uint32_t strd_n = 1 << rxq->strd_num_n; 188 const uint32_t elts_n = wqe_n * strd_n; 189 const uint32_t elts_mask = elts_n - 1; 190 uint32_t elts_idx = rxq->rq_pi & elts_mask; 191 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx]; 192 uint32_t rq_ci = rxq->rq_ci; 193 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask]; 194 uint16_t copied = 0; 195 uint16_t i = 0; 196 197 for (i = 0; i < pkts_n; ++i) { 198 uint16_t strd_cnt; 199 enum mlx5_rqx_code rxq_code; 200 201 if (rxq->consumed_strd == strd_n) { 202 /* Replace WQE if the buffer is still in use. */ 203 mprq_buf_replace(rxq, rq_ci & wqe_mask); 204 /* Advance to the next WQE. */ 205 rxq->consumed_strd = 0; 206 rq_ci++; 207 buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask]; 208 } 209 210 if (!elts[i]->pkt_len) { 211 rxq->consumed_strd = strd_n; 212 rte_pktmbuf_free_seg(elts[i]); 213 #ifdef MLX5_PMD_SOFT_COUNTERS 214 rxq->stats.ipackets -= 1; 215 #endif 216 continue; 217 } 218 strd_cnt = (elts[i]->pkt_len / strd_sz) + 219 ((elts[i]->pkt_len % strd_sz) ? 1 : 0); 220 rxq_code = mprq_buf_to_pkt(rxq, elts[i], elts[i]->pkt_len, 221 buf, rxq->consumed_strd, strd_cnt); 222 rxq->consumed_strd += strd_cnt; 223 if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) { 224 rte_pktmbuf_free_seg(elts[i]); 225 #ifdef MLX5_PMD_SOFT_COUNTERS 226 rxq->stats.ipackets -= 1; 227 rxq->stats.ibytes -= elts[i]->pkt_len; 228 #endif 229 if (rxq_code == MLX5_RXQ_CODE_NOMBUF) { 230 ++rxq->stats.rx_nombuf; 231 break; 232 } 233 if (rxq_code == MLX5_RXQ_CODE_DROPPED) { 234 ++rxq->stats.idropped; 235 continue; 236 } 237 } 238 pkts[copied++] = elts[i]; 239 } 240 rxq->rq_pi += i; 241 rxq->cq_ci += i; 242 rte_io_wmb(); 243 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 244 if (rq_ci != rxq->rq_ci) { 245 rxq->rq_ci = rq_ci; 246 rte_io_wmb(); 247 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 248 } 249 return copied; 250 } 251 252 /** 253 * Receive burst of packets. An errored completion also consumes a mbuf, but the 254 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed 255 * before returning to application. 256 * 257 * @param rxq 258 * Pointer to RX queue structure. 259 * @param[out] pkts 260 * Array to store received packets. 261 * @param pkts_n 262 * Maximum number of packets in array. 263 * @param[out] err 264 * Pointer to a flag. Set non-zero value if pkts array has at least one error 265 * packet to handle. 266 * @param[out] no_cq 267 * Pointer to a boolean. Set true if no new CQE seen. 268 * 269 * @return 270 * Number of packets received including errors (<= pkts_n). 271 */ 272 static inline uint16_t 273 rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 274 uint16_t pkts_n, uint64_t *err, bool *no_cq) 275 { 276 const uint16_t q_n = 1 << rxq->cqe_n; 277 const uint16_t q_mask = q_n - 1; 278 const uint16_t e_n = 1 << rxq->elts_n; 279 const uint16_t e_mask = e_n - 1; 280 volatile struct mlx5_cqe *cq; 281 struct rte_mbuf **elts; 282 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; 283 uint16_t nocmp_n = 0; 284 uint16_t rcvd_pkt = 0; 285 unsigned int cq_idx = rxq->cq_ci & q_mask; 286 unsigned int elts_idx; 287 288 MLX5_ASSERT(rxq->sges_n == 0); 289 MLX5_ASSERT(rxq->cqe_n == rxq->elts_n); 290 cq = &(*rxq->cqes)[cq_idx]; 291 rte_prefetch0(cq); 292 rte_prefetch0(cq + 1); 293 rte_prefetch0(cq + 2); 294 rte_prefetch0(cq + 3); 295 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); 296 mlx5_rx_replenish_bulk_mbuf(rxq); 297 /* See if there're unreturned mbufs from compressed CQE. */ 298 rcvd_pkt = rxq->decompressed; 299 if (rcvd_pkt > 0) { 300 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); 301 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask], 302 pkts, rcvd_pkt); 303 rxq->rq_pi += rcvd_pkt; 304 rxq->decompressed -= rcvd_pkt; 305 pkts += rcvd_pkt; 306 } 307 elts_idx = rxq->rq_pi & e_mask; 308 elts = &(*rxq->elts)[elts_idx]; 309 /* Not to overflow pkts array. */ 310 pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP); 311 /* Not to cross queue end. */ 312 pkts_n = RTE_MIN(pkts_n, q_n - elts_idx); 313 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx); 314 if (!pkts_n) { 315 *no_cq = !rcvd_pkt; 316 return rcvd_pkt; 317 } 318 /* At this point, there shouldn't be any remaining packets. */ 319 MLX5_ASSERT(rxq->decompressed == 0); 320 /* Process all the CQEs */ 321 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); 322 /* If no new CQE seen, return without updating cq_db. */ 323 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) { 324 *no_cq = true; 325 return rcvd_pkt; 326 } 327 /* Update the consumer indexes for non-compressed CQEs. */ 328 MLX5_ASSERT(nocmp_n <= pkts_n); 329 rxq->cq_ci += nocmp_n; 330 rxq->rq_pi += nocmp_n; 331 rcvd_pkt += nocmp_n; 332 /* Decompress the last CQE if compressed. */ 333 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { 334 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); 335 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], 336 &elts[nocmp_n]); 337 rxq->cq_ci += rxq->decompressed; 338 /* Return more packets if needed. */ 339 if (nocmp_n < pkts_n) { 340 uint16_t n = rxq->decompressed; 341 342 n = RTE_MIN(n, pkts_n - nocmp_n); 343 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask], 344 &pkts[nocmp_n], n); 345 rxq->rq_pi += n; 346 rcvd_pkt += n; 347 rxq->decompressed -= n; 348 } 349 } 350 rte_io_wmb(); 351 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 352 *no_cq = !rcvd_pkt; 353 return rcvd_pkt; 354 } 355 356 /** 357 * DPDK callback for vectorized RX. 358 * 359 * @param dpdk_rxq 360 * Generic pointer to RX queue structure. 361 * @param[out] pkts 362 * Array to store received packets. 363 * @param pkts_n 364 * Maximum number of packets in array. 365 * 366 * @return 367 * Number of packets successfully received (<= pkts_n). 368 */ 369 uint16_t 370 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 371 { 372 struct mlx5_rxq_data *rxq = dpdk_rxq; 373 uint16_t nb_rx = 0; 374 uint16_t tn = 0; 375 uint64_t err = 0; 376 bool no_cq = false; 377 378 do { 379 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn, 380 &err, &no_cq); 381 if (unlikely(err | rxq->err_state)) 382 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 383 tn += nb_rx; 384 if (unlikely(no_cq)) 385 break; 386 } while (tn != pkts_n); 387 return tn; 388 } 389 390 /** 391 * Receive burst of packets. An errored completion also consumes a mbuf, but the 392 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed 393 * before returning to application. 394 * 395 * @param rxq 396 * Pointer to RX queue structure. 397 * @param[out] pkts 398 * Array to store received packets. 399 * @param pkts_n 400 * Maximum number of packets in array. 401 * @param[out] err 402 * Pointer to a flag. Set non-zero value if pkts array has at least one error 403 * packet to handle. 404 * @param[out] no_cq 405 * Pointer to a boolean. Set true if no new CQE seen. 406 * 407 * @return 408 * Number of packets received including errors (<= pkts_n). 409 */ 410 static inline uint16_t 411 rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, 412 uint16_t pkts_n, uint64_t *err, bool *no_cq) 413 { 414 const uint16_t q_n = 1 << rxq->cqe_n; 415 const uint16_t q_mask = q_n - 1; 416 const uint16_t wqe_n = 1 << rxq->elts_n; 417 const uint32_t strd_n = 1 << rxq->strd_num_n; 418 const uint32_t elts_n = wqe_n * strd_n; 419 const uint32_t elts_mask = elts_n - 1; 420 volatile struct mlx5_cqe *cq; 421 struct rte_mbuf **elts; 422 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP; 423 uint16_t nocmp_n = 0; 424 uint16_t rcvd_pkt = 0; 425 uint16_t cp_pkt = 0; 426 unsigned int cq_idx = rxq->cq_ci & q_mask; 427 unsigned int elts_idx; 428 429 MLX5_ASSERT(rxq->sges_n == 0); 430 cq = &(*rxq->cqes)[cq_idx]; 431 rte_prefetch0(cq); 432 rte_prefetch0(cq + 1); 433 rte_prefetch0(cq + 2); 434 rte_prefetch0(cq + 3); 435 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST); 436 mlx5_rx_mprq_replenish_bulk_mbuf(rxq); 437 /* See if there're unreturned mbufs from compressed CQE. */ 438 rcvd_pkt = rxq->decompressed; 439 if (rcvd_pkt > 0) { 440 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n); 441 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, rcvd_pkt); 442 rxq->decompressed -= rcvd_pkt; 443 pkts += cp_pkt; 444 } 445 elts_idx = rxq->rq_pi & elts_mask; 446 elts = &(*rxq->elts)[elts_idx]; 447 /* Not to overflow pkts array. */ 448 pkts_n = RTE_ALIGN_FLOOR(pkts_n - cp_pkt, MLX5_VPMD_DESCS_PER_LOOP); 449 /* Not to cross queue end. */ 450 pkts_n = RTE_MIN(pkts_n, elts_n - elts_idx); 451 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx); 452 /* Not to move past the allocated mbufs. */ 453 pkts_n = RTE_MIN(pkts_n, rxq->elts_ci - rxq->rq_pi); 454 if (!pkts_n) { 455 *no_cq = !cp_pkt; 456 return cp_pkt; 457 } 458 /* At this point, there shouldn't be any remaining packets. */ 459 MLX5_ASSERT(rxq->decompressed == 0); 460 /* Process all the CQEs */ 461 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx); 462 /* If no new CQE seen, return without updating cq_db. */ 463 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) { 464 *no_cq = true; 465 return cp_pkt; 466 } 467 /* Update the consumer indexes for non-compressed CQEs. */ 468 MLX5_ASSERT(nocmp_n <= pkts_n); 469 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, nocmp_n); 470 rcvd_pkt += cp_pkt; 471 /* Decompress the last CQE if compressed. */ 472 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) { 473 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP)); 474 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n], 475 &elts[nocmp_n]); 476 /* Return more packets if needed. */ 477 if (nocmp_n < pkts_n) { 478 uint16_t n = rxq->decompressed; 479 480 n = RTE_MIN(n, pkts_n - nocmp_n); 481 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, &pkts[cp_pkt], n); 482 rcvd_pkt += cp_pkt; 483 rxq->decompressed -= n; 484 } 485 } 486 *no_cq = !rcvd_pkt; 487 return rcvd_pkt; 488 } 489 490 /** 491 * DPDK callback for vectorized MPRQ RX. 492 * 493 * @param dpdk_rxq 494 * Generic pointer to RX queue structure. 495 * @param[out] pkts 496 * Array to store received packets. 497 * @param pkts_n 498 * Maximum number of packets in array. 499 * 500 * @return 501 * Number of packets successfully received (<= pkts_n). 502 */ 503 uint16_t 504 mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 505 { 506 struct mlx5_rxq_data *rxq = dpdk_rxq; 507 uint16_t nb_rx = 0; 508 uint16_t tn = 0; 509 uint64_t err = 0; 510 bool no_cq = false; 511 512 do { 513 nb_rx = rxq_burst_mprq_v(rxq, pkts + tn, pkts_n - tn, 514 &err, &no_cq); 515 if (unlikely(err | rxq->err_state)) 516 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx); 517 tn += nb_rx; 518 if (unlikely(no_cq)) 519 break; 520 } while (tn != pkts_n); 521 return tn; 522 } 523 524 /** 525 * Check a RX queue can support vectorized RX. 526 * 527 * @param rxq 528 * Pointer to RX queue. 529 * 530 * @return 531 * 1 if supported, negative errno value if not. 532 */ 533 int __rte_cold 534 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) 535 { 536 struct mlx5_rxq_ctrl *ctrl = 537 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 538 539 if (!ctrl->priv->config.rx_vec_en || rxq->sges_n != 0) 540 return -ENOTSUP; 541 if (rxq->lro) 542 return -ENOTSUP; 543 return 1; 544 } 545 546 /** 547 * Check a device can support vectorized RX. 548 * 549 * @param dev 550 * Pointer to Ethernet device. 551 * 552 * @return 553 * 1 if supported, negative errno value if not. 554 */ 555 int __rte_cold 556 mlx5_check_vec_rx_support(struct rte_eth_dev *dev) 557 { 558 struct mlx5_priv *priv = dev->data->dev_private; 559 uint32_t i; 560 561 if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) 562 return -ENOTSUP; 563 if (!priv->config.rx_vec_en) 564 return -ENOTSUP; 565 /* All the configured queues should support. */ 566 for (i = 0; i < priv->rxqs_n; ++i) { 567 struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; 568 569 if (!rxq) 570 continue; 571 if (mlx5_rxq_check_vec_support(rxq) < 0) 572 break; 573 } 574 if (i != priv->rxqs_n) 575 return -ENOTSUP; 576 return 1; 577 } 578