1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 6WIND S.A. 3 * Copyright 2021 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 #include <rte_mbuf.h> 11 #include <rte_mempool.h> 12 #include <rte_prefetch.h> 13 #include <rte_common.h> 14 #include <rte_branch_prediction.h> 15 #include <rte_ether.h> 16 #include <rte_cycles.h> 17 #include <rte_flow.h> 18 19 #include <mlx5_prm.h> 20 #include <mlx5_common.h> 21 22 #include "mlx5_autoconf.h" 23 #include "mlx5_defs.h" 24 #include "mlx5.h" 25 #include "mlx5_mr.h" 26 #include "mlx5_utils.h" 27 #include "mlx5_rxtx.h" 28 #include "mlx5_rx.h" 29 30 31 static __rte_always_inline uint32_t 32 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 33 volatile struct mlx5_mini_cqe8 *mcqe); 34 35 static __rte_always_inline int 36 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 37 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe); 38 39 static __rte_always_inline uint32_t 40 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe); 41 42 static __rte_always_inline void 43 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, 44 volatile struct mlx5_cqe *cqe, 45 volatile struct mlx5_mini_cqe8 *mcqe); 46 47 static inline void 48 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp, 49 volatile struct mlx5_cqe *__rte_restrict cqe, 50 uint32_t phcsum, uint8_t l4_type); 51 52 static inline void 53 mlx5_lro_update_hdr(uint8_t *__rte_restrict padd, 54 volatile struct mlx5_cqe *__rte_restrict cqe, 55 volatile struct mlx5_mini_cqe8 *mcqe, 56 struct mlx5_rxq_data *rxq, uint32_t len); 57 58 59 /** 60 * Internal function to compute the number of used descriptors in an RX queue. 61 * 62 * @param rxq 63 * The Rx queue. 64 * 65 * @return 66 * The number of used Rx descriptor. 67 */ 68 static uint32_t 69 rx_queue_count(struct mlx5_rxq_data *rxq) 70 { 71 struct rxq_zip *zip = &rxq->zip; 72 volatile struct mlx5_cqe *cqe; 73 const unsigned int cqe_n = (1 << rxq->cqe_n); 74 const unsigned int sges_n = (1 << rxq->sges_n); 75 const unsigned int elts_n = (1 << rxq->elts_n); 76 const unsigned int strd_n = (1 << rxq->strd_num_n); 77 const unsigned int cqe_cnt = cqe_n - 1; 78 unsigned int cq_ci, used; 79 80 /* if we are processing a compressed cqe */ 81 if (zip->ai) { 82 used = zip->cqe_cnt - zip->ai; 83 cq_ci = zip->cq_ci; 84 } else { 85 used = 0; 86 cq_ci = rxq->cq_ci; 87 } 88 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; 89 while (check_cqe(cqe, cqe_n, cq_ci) != MLX5_CQE_STATUS_HW_OWN) { 90 int8_t op_own; 91 unsigned int n; 92 93 op_own = cqe->op_own; 94 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) 95 n = rte_be_to_cpu_32(cqe->byte_cnt); 96 else 97 n = 1; 98 cq_ci += n; 99 used += n; 100 cqe = &(*rxq->cqes)[cq_ci & cqe_cnt]; 101 } 102 used = RTE_MIN(used * sges_n, elts_n * strd_n); 103 return used; 104 } 105 106 /** 107 * DPDK callback to check the status of a Rx descriptor. 108 * 109 * @param rx_queue 110 * The Rx queue. 111 * @param[in] offset 112 * The index of the descriptor in the ring. 113 * 114 * @return 115 * The status of the Rx descriptor. 116 */ 117 int 118 mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset) 119 { 120 struct mlx5_rxq_data *rxq = rx_queue; 121 struct mlx5_rxq_ctrl *rxq_ctrl = 122 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 123 struct rte_eth_dev *dev = ETH_DEV(rxq_ctrl->priv); 124 125 if (dev->rx_pkt_burst == NULL || 126 dev->rx_pkt_burst == removed_rx_burst) { 127 rte_errno = ENOTSUP; 128 return -rte_errno; 129 } 130 if (offset >= (1 << rxq->cqe_n)) { 131 rte_errno = EINVAL; 132 return -rte_errno; 133 } 134 if (offset < rx_queue_count(rxq)) 135 return RTE_ETH_RX_DESC_DONE; 136 return RTE_ETH_RX_DESC_AVAIL; 137 } 138 139 /** 140 * DPDK callback to get the RX queue information. 141 * 142 * @param dev 143 * Pointer to the device structure. 144 * 145 * @param rx_queue_id 146 * Rx queue identificator. 147 * 148 * @param qinfo 149 * Pointer to the RX queue information structure. 150 * 151 * @return 152 * None. 153 */ 154 155 void 156 mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, 157 struct rte_eth_rxq_info *qinfo) 158 { 159 struct mlx5_priv *priv = dev->data->dev_private; 160 struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id]; 161 struct mlx5_rxq_ctrl *rxq_ctrl = 162 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 163 164 if (!rxq) 165 return; 166 qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ? 167 rxq->mprq_mp : rxq->mp; 168 qinfo->conf.rx_thresh.pthresh = 0; 169 qinfo->conf.rx_thresh.hthresh = 0; 170 qinfo->conf.rx_thresh.wthresh = 0; 171 qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh; 172 qinfo->conf.rx_drop_en = 1; 173 qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1; 174 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 175 qinfo->scattered_rx = dev->data->scattered_rx; 176 qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ? 177 (1 << rxq->elts_n) * (1 << rxq->strd_num_n) : 178 (1 << rxq->elts_n); 179 } 180 181 /** 182 * DPDK callback to get the RX packet burst mode information. 183 * 184 * @param dev 185 * Pointer to the device structure. 186 * 187 * @param rx_queue_id 188 * Rx queue identificatior. 189 * 190 * @param mode 191 * Pointer to the burts mode information. 192 * 193 * @return 194 * 0 as success, -EINVAL as failure. 195 */ 196 int 197 mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, 198 uint16_t rx_queue_id __rte_unused, 199 struct rte_eth_burst_mode *mode) 200 { 201 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 202 struct mlx5_priv *priv = dev->data->dev_private; 203 struct mlx5_rxq_data *rxq; 204 205 rxq = (*priv->rxqs)[rx_queue_id]; 206 if (!rxq) { 207 rte_errno = EINVAL; 208 return -rte_errno; 209 } 210 if (pkt_burst == mlx5_rx_burst) { 211 snprintf(mode->info, sizeof(mode->info), "%s", "Scalar"); 212 } else if (pkt_burst == mlx5_rx_burst_mprq) { 213 snprintf(mode->info, sizeof(mode->info), "%s", "Multi-Packet RQ"); 214 } else if (pkt_burst == mlx5_rx_burst_vec) { 215 #if defined RTE_ARCH_X86_64 216 snprintf(mode->info, sizeof(mode->info), "%s", "Vector SSE"); 217 #elif defined RTE_ARCH_ARM64 218 snprintf(mode->info, sizeof(mode->info), "%s", "Vector Neon"); 219 #elif defined RTE_ARCH_PPC_64 220 snprintf(mode->info, sizeof(mode->info), "%s", "Vector AltiVec"); 221 #else 222 return -EINVAL; 223 #endif 224 } else if (pkt_burst == mlx5_rx_burst_mprq_vec) { 225 #if defined RTE_ARCH_X86_64 226 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector SSE"); 227 #elif defined RTE_ARCH_ARM64 228 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector Neon"); 229 #elif defined RTE_ARCH_PPC_64 230 snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ Vector AltiVec"); 231 #else 232 return -EINVAL; 233 #endif 234 } else { 235 return -EINVAL; 236 } 237 return 0; 238 } 239 240 /** 241 * DPDK callback to get the number of used descriptors in a RX queue. 242 * 243 * @param dev 244 * Pointer to the device structure. 245 * 246 * @param rx_queue_id 247 * The Rx queue. 248 * 249 * @return 250 * The number of used rx descriptor. 251 * -EINVAL if the queue is invalid 252 */ 253 uint32_t 254 mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) 255 { 256 struct mlx5_priv *priv = dev->data->dev_private; 257 struct mlx5_rxq_data *rxq; 258 259 if (dev->rx_pkt_burst == NULL || 260 dev->rx_pkt_burst == removed_rx_burst) { 261 rte_errno = ENOTSUP; 262 return -rte_errno; 263 } 264 rxq = (*priv->rxqs)[rx_queue_id]; 265 if (!rxq) { 266 rte_errno = EINVAL; 267 return -rte_errno; 268 } 269 return rx_queue_count(rxq); 270 } 271 272 int mlx5_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) 273 { 274 struct mlx5_rxq_data *rxq = rx_queue; 275 const unsigned int cqe_num = 1 << rxq->cqe_n; 276 const unsigned int cqe_mask = cqe_num - 1; 277 const uint16_t idx = rxq->cq_ci & cqe_num; 278 volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask]; 279 280 if (unlikely(rxq->cqes == NULL)) { 281 rte_errno = EINVAL; 282 return -rte_errno; 283 } 284 pmc->addr = &cqe->op_own; 285 pmc->val = !!idx; 286 pmc->mask = MLX5_CQE_OWNER_MASK; 287 pmc->size = sizeof(uint8_t); 288 return 0; 289 } 290 291 /** 292 * Translate RX completion flags to packet type. 293 * 294 * @param[in] rxq 295 * Pointer to RX queue structure. 296 * @param[in] cqe 297 * Pointer to CQE. 298 * 299 * @note: fix mlx5_dev_supported_ptypes_get() if any change here. 300 * 301 * @return 302 * Packet type for struct rte_mbuf. 303 */ 304 static inline uint32_t 305 rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 306 volatile struct mlx5_mini_cqe8 *mcqe) 307 { 308 uint8_t idx; 309 uint8_t ptype; 310 uint8_t pinfo = (cqe->pkt_info & 0x3) << 6; 311 312 /* Get l3/l4 header from mini-CQE in case L3/L4 format*/ 313 if (mcqe == NULL || 314 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX) 315 ptype = (cqe->hdr_type_etc & 0xfc00) >> 10; 316 else 317 ptype = mcqe->hdr_type >> 2; 318 /* 319 * The index to the array should have: 320 * bit[1:0] = l3_hdr_type 321 * bit[4:2] = l4_hdr_type 322 * bit[5] = ip_frag 323 * bit[6] = tunneled 324 * bit[7] = outer_l3_type 325 */ 326 idx = pinfo | ptype; 327 return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6)); 328 } 329 330 /** 331 * Initialize Rx WQ and indexes. 332 * 333 * @param[in] rxq 334 * Pointer to RX queue structure. 335 */ 336 void 337 mlx5_rxq_initialize(struct mlx5_rxq_data *rxq) 338 { 339 const unsigned int wqe_n = 1 << rxq->elts_n; 340 unsigned int i; 341 342 for (i = 0; (i != wqe_n); ++i) { 343 volatile struct mlx5_wqe_data_seg *scat; 344 uintptr_t addr; 345 uint32_t byte_count; 346 347 if (mlx5_rxq_mprq_enabled(rxq)) { 348 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i]; 349 350 scat = &((volatile struct mlx5_wqe_mprq *) 351 rxq->wqes)[i].dseg; 352 addr = (uintptr_t)mlx5_mprq_buf_addr(buf, 353 1 << rxq->strd_num_n); 354 byte_count = (1 << rxq->strd_sz_n) * 355 (1 << rxq->strd_num_n); 356 } else { 357 struct rte_mbuf *buf = (*rxq->elts)[i]; 358 359 scat = &((volatile struct mlx5_wqe_data_seg *) 360 rxq->wqes)[i]; 361 addr = rte_pktmbuf_mtod(buf, uintptr_t); 362 byte_count = DATA_LEN(buf); 363 } 364 /* scat->addr must be able to store a pointer. */ 365 MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t)); 366 *scat = (struct mlx5_wqe_data_seg){ 367 .addr = rte_cpu_to_be_64(addr), 368 .byte_count = rte_cpu_to_be_32(byte_count), 369 .lkey = mlx5_rx_addr2mr(rxq, addr), 370 }; 371 } 372 rxq->consumed_strd = 0; 373 rxq->decompressed = 0; 374 rxq->rq_pi = 0; 375 rxq->zip = (struct rxq_zip){ 376 .ai = 0, 377 }; 378 rxq->elts_ci = mlx5_rxq_mprq_enabled(rxq) ? 379 (wqe_n >> rxq->sges_n) * (1 << rxq->strd_num_n) : 0; 380 /* Update doorbell counter. */ 381 rxq->rq_ci = wqe_n >> rxq->sges_n; 382 rte_io_wmb(); 383 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 384 } 385 386 /** 387 * Handle a Rx error. 388 * The function inserts the RQ state to reset when the first error CQE is 389 * shown, then drains the CQ by the caller function loop. When the CQ is empty, 390 * it moves the RQ state to ready and initializes the RQ. 391 * Next CQE identification and error counting are in the caller responsibility. 392 * 393 * @param[in] rxq 394 * Pointer to RX queue structure. 395 * @param[in] vec 396 * 1 when called from vectorized Rx burst, need to prepare mbufs for the RQ. 397 * 0 when called from non-vectorized Rx burst. 398 * 399 * @return 400 * -1 in case of recovery error, otherwise the CQE status. 401 */ 402 int 403 mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec) 404 { 405 const uint16_t cqe_n = 1 << rxq->cqe_n; 406 const uint16_t cqe_mask = cqe_n - 1; 407 const uint16_t wqe_n = 1 << rxq->elts_n; 408 const uint16_t strd_n = 1 << rxq->strd_num_n; 409 struct mlx5_rxq_ctrl *rxq_ctrl = 410 container_of(rxq, struct mlx5_rxq_ctrl, rxq); 411 union { 412 volatile struct mlx5_cqe *cqe; 413 volatile struct mlx5_err_cqe *err_cqe; 414 } u = { 415 .cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask], 416 }; 417 struct mlx5_mp_arg_queue_state_modify sm; 418 int ret; 419 420 switch (rxq->err_state) { 421 case MLX5_RXQ_ERR_STATE_NO_ERROR: 422 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET; 423 /* Fall-through */ 424 case MLX5_RXQ_ERR_STATE_NEED_RESET: 425 sm.is_wq = 1; 426 sm.queue_id = rxq->idx; 427 sm.state = IBV_WQS_RESET; 428 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), &sm)) 429 return -1; 430 if (rxq_ctrl->dump_file_n < 431 rxq_ctrl->priv->config.max_dump_files_num) { 432 MKSTR(err_str, "Unexpected CQE error syndrome " 433 "0x%02x CQN = %u RQN = %u wqe_counter = %u" 434 " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome, 435 rxq->cqn, rxq_ctrl->wqn, 436 rte_be_to_cpu_16(u.err_cqe->wqe_counter), 437 rxq->rq_ci << rxq->sges_n, rxq->cq_ci); 438 MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u", 439 rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc()); 440 mlx5_dump_debug_information(name, NULL, err_str, 0); 441 mlx5_dump_debug_information(name, "MLX5 Error CQ:", 442 (const void *)((uintptr_t) 443 rxq->cqes), 444 sizeof(*u.cqe) * cqe_n); 445 mlx5_dump_debug_information(name, "MLX5 Error RQ:", 446 (const void *)((uintptr_t) 447 rxq->wqes), 448 16 * wqe_n); 449 rxq_ctrl->dump_file_n++; 450 } 451 rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY; 452 /* Fall-through */ 453 case MLX5_RXQ_ERR_STATE_NEED_READY: 454 ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci); 455 if (ret == MLX5_CQE_STATUS_HW_OWN) { 456 rte_io_wmb(); 457 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 458 rte_io_wmb(); 459 /* 460 * The RQ consumer index must be zeroed while moving 461 * from RESET state to RDY state. 462 */ 463 *rxq->rq_db = rte_cpu_to_be_32(0); 464 rte_io_wmb(); 465 sm.is_wq = 1; 466 sm.queue_id = rxq->idx; 467 sm.state = IBV_WQS_RDY; 468 if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv), 469 &sm)) 470 return -1; 471 if (vec) { 472 const uint32_t elts_n = 473 mlx5_rxq_mprq_enabled(rxq) ? 474 wqe_n * strd_n : wqe_n; 475 const uint32_t e_mask = elts_n - 1; 476 uint32_t elts_ci = 477 mlx5_rxq_mprq_enabled(rxq) ? 478 rxq->elts_ci : rxq->rq_ci; 479 uint32_t elt_idx; 480 struct rte_mbuf **elt; 481 int i; 482 unsigned int n = elts_n - (elts_ci - 483 rxq->rq_pi); 484 485 for (i = 0; i < (int)n; ++i) { 486 elt_idx = (elts_ci + i) & e_mask; 487 elt = &(*rxq->elts)[elt_idx]; 488 *elt = rte_mbuf_raw_alloc(rxq->mp); 489 if (!*elt) { 490 for (i--; i >= 0; --i) { 491 elt_idx = (elts_ci + 492 i) & elts_n; 493 elt = &(*rxq->elts) 494 [elt_idx]; 495 rte_pktmbuf_free_seg 496 (*elt); 497 } 498 return -1; 499 } 500 } 501 for (i = 0; i < (int)elts_n; ++i) { 502 elt = &(*rxq->elts)[i]; 503 DATA_LEN(*elt) = 504 (uint16_t)((*elt)->buf_len - 505 rte_pktmbuf_headroom(*elt)); 506 } 507 /* Padding with a fake mbuf for vec Rx. */ 508 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i) 509 (*rxq->elts)[elts_n + i] = 510 &rxq->fake_mbuf; 511 } 512 mlx5_rxq_initialize(rxq); 513 rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR; 514 } 515 return ret; 516 default: 517 return -1; 518 } 519 } 520 521 /** 522 * Get size of the next packet for a given CQE. For compressed CQEs, the 523 * consumer index is updated only once all packets of the current one have 524 * been processed. 525 * 526 * @param rxq 527 * Pointer to RX queue. 528 * @param cqe 529 * CQE to process. 530 * @param[out] mcqe 531 * Store pointer to mini-CQE if compressed. Otherwise, the pointer is not 532 * written. 533 * 534 * @return 535 * 0 in case of empty CQE, otherwise the packet size in bytes. 536 */ 537 static inline int 538 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, 539 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe) 540 { 541 struct rxq_zip *zip = &rxq->zip; 542 uint16_t cqe_n = cqe_cnt + 1; 543 int len; 544 uint16_t idx, end; 545 546 do { 547 len = 0; 548 /* Process compressed data in the CQE and mini arrays. */ 549 if (zip->ai) { 550 volatile struct mlx5_mini_cqe8 (*mc)[8] = 551 (volatile struct mlx5_mini_cqe8 (*)[8]) 552 (uintptr_t)(&(*rxq->cqes)[zip->ca & 553 cqe_cnt].pkt_info); 554 len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt & 555 rxq->byte_mask); 556 *mcqe = &(*mc)[zip->ai & 7]; 557 if ((++zip->ai & 7) == 0) { 558 /* Invalidate consumed CQEs */ 559 idx = zip->ca; 560 end = zip->na; 561 while (idx != end) { 562 (*rxq->cqes)[idx & cqe_cnt].op_own = 563 MLX5_CQE_INVALIDATE; 564 ++idx; 565 } 566 /* 567 * Increment consumer index to skip the number 568 * of CQEs consumed. Hardware leaves holes in 569 * the CQ ring for software use. 570 */ 571 zip->ca = zip->na; 572 zip->na += 8; 573 } 574 if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) { 575 /* Invalidate the rest */ 576 idx = zip->ca; 577 end = zip->cq_ci; 578 579 while (idx != end) { 580 (*rxq->cqes)[idx & cqe_cnt].op_own = 581 MLX5_CQE_INVALIDATE; 582 ++idx; 583 } 584 rxq->cq_ci = zip->cq_ci; 585 zip->ai = 0; 586 } 587 /* 588 * No compressed data, get next CQE and verify if it is 589 * compressed. 590 */ 591 } else { 592 int ret; 593 int8_t op_own; 594 uint32_t cq_ci; 595 596 ret = check_cqe(cqe, cqe_n, rxq->cq_ci); 597 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 598 if (unlikely(ret == MLX5_CQE_STATUS_ERR || 599 rxq->err_state)) { 600 ret = mlx5_rx_err_handle(rxq, 0); 601 if (ret == MLX5_CQE_STATUS_HW_OWN || 602 ret == -1) 603 return 0; 604 } else { 605 return 0; 606 } 607 } 608 /* 609 * Introduce the local variable to have queue cq_ci 610 * index in queue structure always consistent with 611 * actual CQE boundary (not pointing to the middle 612 * of compressed CQE session). 613 */ 614 cq_ci = rxq->cq_ci + 1; 615 op_own = cqe->op_own; 616 if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) { 617 volatile struct mlx5_mini_cqe8 (*mc)[8] = 618 (volatile struct mlx5_mini_cqe8 (*)[8]) 619 (uintptr_t)(&(*rxq->cqes) 620 [cq_ci & cqe_cnt].pkt_info); 621 622 /* Fix endianness. */ 623 zip->cqe_cnt = rte_be_to_cpu_32(cqe->byte_cnt); 624 /* 625 * Current mini array position is the one 626 * returned by check_cqe64(). 627 * 628 * If completion comprises several mini arrays, 629 * as a special case the second one is located 630 * 7 CQEs after the initial CQE instead of 8 631 * for subsequent ones. 632 */ 633 zip->ca = cq_ci; 634 zip->na = zip->ca + 7; 635 /* Compute the next non compressed CQE. */ 636 zip->cq_ci = rxq->cq_ci + zip->cqe_cnt; 637 /* Get packet size to return. */ 638 len = rte_be_to_cpu_32((*mc)[0].byte_cnt & 639 rxq->byte_mask); 640 *mcqe = &(*mc)[0]; 641 zip->ai = 1; 642 /* Prefetch all to be invalidated */ 643 idx = zip->ca; 644 end = zip->cq_ci; 645 while (idx != end) { 646 rte_prefetch0(&(*rxq->cqes)[(idx) & 647 cqe_cnt]); 648 ++idx; 649 } 650 } else { 651 rxq->cq_ci = cq_ci; 652 len = rte_be_to_cpu_32(cqe->byte_cnt); 653 } 654 } 655 if (unlikely(rxq->err_state)) { 656 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 657 ++rxq->stats.idropped; 658 } else { 659 return len; 660 } 661 } while (1); 662 } 663 664 /** 665 * Translate RX completion flags to offload flags. 666 * 667 * @param[in] cqe 668 * Pointer to CQE. 669 * 670 * @return 671 * Offload flags (ol_flags) for struct rte_mbuf. 672 */ 673 static inline uint32_t 674 rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe) 675 { 676 uint32_t ol_flags = 0; 677 uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc); 678 679 ol_flags = 680 TRANSPOSE(flags, 681 MLX5_CQE_RX_L3_HDR_VALID, 682 PKT_RX_IP_CKSUM_GOOD) | 683 TRANSPOSE(flags, 684 MLX5_CQE_RX_L4_HDR_VALID, 685 PKT_RX_L4_CKSUM_GOOD); 686 return ol_flags; 687 } 688 689 /** 690 * Fill in mbuf fields from RX completion flags. 691 * Note that pkt->ol_flags should be initialized outside of this function. 692 * 693 * @param rxq 694 * Pointer to RX queue. 695 * @param pkt 696 * mbuf to fill. 697 * @param cqe 698 * CQE to process. 699 * @param rss_hash_res 700 * Packet RSS Hash result. 701 */ 702 static inline void 703 rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, 704 volatile struct mlx5_cqe *cqe, 705 volatile struct mlx5_mini_cqe8 *mcqe) 706 { 707 /* Update packet information. */ 708 pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe, mcqe); 709 710 if (rxq->rss_hash) { 711 uint32_t rss_hash_res = 0; 712 713 /* If compressed, take hash result from mini-CQE. */ 714 if (mcqe == NULL || 715 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_HASH) 716 rss_hash_res = rte_be_to_cpu_32(cqe->rx_hash_res); 717 else 718 rss_hash_res = rte_be_to_cpu_32(mcqe->rx_hash_result); 719 if (rss_hash_res) { 720 pkt->hash.rss = rss_hash_res; 721 pkt->ol_flags |= PKT_RX_RSS_HASH; 722 } 723 } 724 if (rxq->mark) { 725 uint32_t mark = 0; 726 727 /* If compressed, take flow tag from mini-CQE. */ 728 if (mcqe == NULL || 729 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_FTAG_STRIDX) 730 mark = cqe->sop_drop_qpn; 731 else 732 mark = ((mcqe->byte_cnt_flow & 0xff) << 8) | 733 (mcqe->flow_tag_high << 16); 734 if (MLX5_FLOW_MARK_IS_VALID(mark)) { 735 pkt->ol_flags |= PKT_RX_FDIR; 736 if (mark != RTE_BE32(MLX5_FLOW_MARK_DEFAULT)) { 737 pkt->ol_flags |= PKT_RX_FDIR_ID; 738 pkt->hash.fdir.hi = mlx5_flow_mark_get(mark); 739 } 740 } 741 } 742 if (rxq->dynf_meta) { 743 uint32_t meta = rte_be_to_cpu_32(cqe->flow_table_metadata >> 744 __builtin_popcount(rxq->flow_meta_port_mask)) & 745 rxq->flow_meta_port_mask; 746 747 if (meta) { 748 pkt->ol_flags |= rxq->flow_meta_mask; 749 *RTE_MBUF_DYNFIELD(pkt, rxq->flow_meta_offset, 750 uint32_t *) = meta; 751 } 752 } 753 if (rxq->csum) 754 pkt->ol_flags |= rxq_cq_to_ol_flags(cqe); 755 if (rxq->vlan_strip) { 756 bool vlan_strip; 757 758 if (mcqe == NULL || 759 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX) 760 vlan_strip = cqe->hdr_type_etc & 761 RTE_BE16(MLX5_CQE_VLAN_STRIPPED); 762 else 763 vlan_strip = mcqe->hdr_type & 764 RTE_BE16(MLX5_CQE_VLAN_STRIPPED); 765 if (vlan_strip) { 766 pkt->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 767 pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info); 768 } 769 } 770 if (rxq->hw_timestamp) { 771 uint64_t ts = rte_be_to_cpu_64(cqe->timestamp); 772 773 if (rxq->rt_timestamp) 774 ts = mlx5_txpp_convert_rx_ts(rxq->sh, ts); 775 mlx5_timestamp_set(pkt, rxq->timestamp_offset, ts); 776 pkt->ol_flags |= rxq->timestamp_rx_flag; 777 } 778 } 779 780 /** 781 * DPDK callback for RX. 782 * 783 * @param dpdk_rxq 784 * Generic pointer to RX queue structure. 785 * @param[out] pkts 786 * Array to store received packets. 787 * @param pkts_n 788 * Maximum number of packets in array. 789 * 790 * @return 791 * Number of packets successfully received (<= pkts_n). 792 */ 793 uint16_t 794 mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 795 { 796 struct mlx5_rxq_data *rxq = dpdk_rxq; 797 const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1; 798 const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1; 799 const unsigned int sges_n = rxq->sges_n; 800 struct rte_mbuf *pkt = NULL; 801 struct rte_mbuf *seg = NULL; 802 volatile struct mlx5_cqe *cqe = 803 &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 804 unsigned int i = 0; 805 unsigned int rq_ci = rxq->rq_ci << sges_n; 806 int len = 0; /* keep its value across iterations. */ 807 808 while (pkts_n) { 809 unsigned int idx = rq_ci & wqe_cnt; 810 volatile struct mlx5_wqe_data_seg *wqe = 811 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[idx]; 812 struct rte_mbuf *rep = (*rxq->elts)[idx]; 813 volatile struct mlx5_mini_cqe8 *mcqe = NULL; 814 815 if (pkt) 816 NEXT(seg) = rep; 817 seg = rep; 818 rte_prefetch0(seg); 819 rte_prefetch0(cqe); 820 rte_prefetch0(wqe); 821 /* Allocate the buf from the same pool. */ 822 rep = rte_mbuf_raw_alloc(seg->pool); 823 if (unlikely(rep == NULL)) { 824 ++rxq->stats.rx_nombuf; 825 if (!pkt) { 826 /* 827 * no buffers before we even started, 828 * bail out silently. 829 */ 830 break; 831 } 832 while (pkt != seg) { 833 MLX5_ASSERT(pkt != (*rxq->elts)[idx]); 834 rep = NEXT(pkt); 835 NEXT(pkt) = NULL; 836 NB_SEGS(pkt) = 1; 837 rte_mbuf_raw_free(pkt); 838 pkt = rep; 839 } 840 rq_ci >>= sges_n; 841 ++rq_ci; 842 rq_ci <<= sges_n; 843 break; 844 } 845 if (!pkt) { 846 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt]; 847 len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe); 848 if (!len) { 849 rte_mbuf_raw_free(rep); 850 break; 851 } 852 pkt = seg; 853 MLX5_ASSERT(len >= (rxq->crc_present << 2)); 854 pkt->ol_flags &= EXT_ATTACHED_MBUF; 855 rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe); 856 if (rxq->crc_present) 857 len -= RTE_ETHER_CRC_LEN; 858 PKT_LEN(pkt) = len; 859 if (cqe->lro_num_seg > 1) { 860 mlx5_lro_update_hdr 861 (rte_pktmbuf_mtod(pkt, uint8_t *), cqe, 862 mcqe, rxq, len); 863 pkt->ol_flags |= PKT_RX_LRO; 864 pkt->tso_segsz = len / cqe->lro_num_seg; 865 } 866 } 867 DATA_LEN(rep) = DATA_LEN(seg); 868 PKT_LEN(rep) = PKT_LEN(seg); 869 SET_DATA_OFF(rep, DATA_OFF(seg)); 870 PORT(rep) = PORT(seg); 871 (*rxq->elts)[idx] = rep; 872 /* 873 * Fill NIC descriptor with the new buffer. The lkey and size 874 * of the buffers are already known, only the buffer address 875 * changes. 876 */ 877 wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t)); 878 /* If there's only one MR, no need to replace LKey in WQE. */ 879 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) 880 wqe->lkey = mlx5_rx_mb2mr(rxq, rep); 881 if (len > DATA_LEN(seg)) { 882 len -= DATA_LEN(seg); 883 ++NB_SEGS(pkt); 884 ++rq_ci; 885 continue; 886 } 887 DATA_LEN(seg) = len; 888 #ifdef MLX5_PMD_SOFT_COUNTERS 889 /* Increment bytes counter. */ 890 rxq->stats.ibytes += PKT_LEN(pkt); 891 #endif 892 /* Return packet. */ 893 *(pkts++) = pkt; 894 pkt = NULL; 895 --pkts_n; 896 ++i; 897 /* Align consumer index to the next stride. */ 898 rq_ci >>= sges_n; 899 ++rq_ci; 900 rq_ci <<= sges_n; 901 } 902 if (unlikely(i == 0 && ((rq_ci >> sges_n) == rxq->rq_ci))) 903 return 0; 904 /* Update the consumer index. */ 905 rxq->rq_ci = rq_ci >> sges_n; 906 rte_io_wmb(); 907 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 908 rte_io_wmb(); 909 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 910 #ifdef MLX5_PMD_SOFT_COUNTERS 911 /* Increment packets counter. */ 912 rxq->stats.ipackets += i; 913 #endif 914 return i; 915 } 916 917 /** 918 * Update LRO packet TCP header. 919 * The HW LRO feature doesn't update the TCP header after coalescing the 920 * TCP segments but supplies information in CQE to fill it by SW. 921 * 922 * @param tcp 923 * Pointer to the TCP header. 924 * @param cqe 925 * Pointer to the completion entry. 926 * @param phcsum 927 * The L3 pseudo-header checksum. 928 */ 929 static inline void 930 mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp, 931 volatile struct mlx5_cqe *__rte_restrict cqe, 932 uint32_t phcsum, uint8_t l4_type) 933 { 934 /* 935 * The HW calculates only the TCP payload checksum, need to complete 936 * the TCP header checksum and the L3 pseudo-header checksum. 937 */ 938 uint32_t csum = phcsum + cqe->csum; 939 940 if (l4_type == MLX5_L4_HDR_TYPE_TCP_EMPTY_ACK || 941 l4_type == MLX5_L4_HDR_TYPE_TCP_WITH_ACL) { 942 tcp->tcp_flags |= RTE_TCP_ACK_FLAG; 943 tcp->recv_ack = cqe->lro_ack_seq_num; 944 tcp->rx_win = cqe->lro_tcp_win; 945 } 946 if (cqe->lro_tcppsh_abort_dupack & MLX5_CQE_LRO_PUSH_MASK) 947 tcp->tcp_flags |= RTE_TCP_PSH_FLAG; 948 tcp->cksum = 0; 949 csum += rte_raw_cksum(tcp, (tcp->data_off >> 4) * 4); 950 csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff); 951 csum = (~csum) & 0xffff; 952 if (csum == 0) 953 csum = 0xffff; 954 tcp->cksum = csum; 955 } 956 957 /** 958 * Update LRO packet headers. 959 * The HW LRO feature doesn't update the L3/TCP headers after coalescing the 960 * TCP segments but supply information in CQE to fill it by SW. 961 * 962 * @param padd 963 * The packet address. 964 * @param cqe 965 * Pointer to the completion entry. 966 * @param len 967 * The packet length. 968 */ 969 static inline void 970 mlx5_lro_update_hdr(uint8_t *__rte_restrict padd, 971 volatile struct mlx5_cqe *__rte_restrict cqe, 972 volatile struct mlx5_mini_cqe8 *mcqe, 973 struct mlx5_rxq_data *rxq, uint32_t len) 974 { 975 union { 976 struct rte_ether_hdr *eth; 977 struct rte_vlan_hdr *vlan; 978 struct rte_ipv4_hdr *ipv4; 979 struct rte_ipv6_hdr *ipv6; 980 struct rte_tcp_hdr *tcp; 981 uint8_t *hdr; 982 } h = { 983 .hdr = padd, 984 }; 985 uint16_t proto = h.eth->ether_type; 986 uint32_t phcsum; 987 uint8_t l4_type; 988 989 h.eth++; 990 while (proto == RTE_BE16(RTE_ETHER_TYPE_VLAN) || 991 proto == RTE_BE16(RTE_ETHER_TYPE_QINQ)) { 992 proto = h.vlan->eth_proto; 993 h.vlan++; 994 } 995 if (proto == RTE_BE16(RTE_ETHER_TYPE_IPV4)) { 996 h.ipv4->time_to_live = cqe->lro_min_ttl; 997 h.ipv4->total_length = rte_cpu_to_be_16(len - (h.hdr - padd)); 998 h.ipv4->hdr_checksum = 0; 999 h.ipv4->hdr_checksum = rte_ipv4_cksum(h.ipv4); 1000 phcsum = rte_ipv4_phdr_cksum(h.ipv4, 0); 1001 h.ipv4++; 1002 } else { 1003 h.ipv6->hop_limits = cqe->lro_min_ttl; 1004 h.ipv6->payload_len = rte_cpu_to_be_16(len - (h.hdr - padd) - 1005 sizeof(*h.ipv6)); 1006 phcsum = rte_ipv6_phdr_cksum(h.ipv6, 0); 1007 h.ipv6++; 1008 } 1009 if (mcqe == NULL || 1010 rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX) 1011 l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) & 1012 MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT; 1013 else 1014 l4_type = (rte_be_to_cpu_16(mcqe->hdr_type) & 1015 MLX5_CQE_L4_TYPE_MASK) >> MLX5_CQE_L4_TYPE_SHIFT; 1016 mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum, l4_type); 1017 } 1018 1019 void 1020 mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque) 1021 { 1022 struct mlx5_mprq_buf *buf = opaque; 1023 1024 if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) == 1) { 1025 rte_mempool_put(buf->mp, buf); 1026 } else if (unlikely(__atomic_sub_fetch(&buf->refcnt, 1, 1027 __ATOMIC_RELAXED) == 0)) { 1028 __atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED); 1029 rte_mempool_put(buf->mp, buf); 1030 } 1031 } 1032 1033 void 1034 mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf) 1035 { 1036 mlx5_mprq_buf_free_cb(NULL, buf); 1037 } 1038 1039 /** 1040 * DPDK callback for RX with Multi-Packet RQ support. 1041 * 1042 * @param dpdk_rxq 1043 * Generic pointer to RX queue structure. 1044 * @param[out] pkts 1045 * Array to store received packets. 1046 * @param pkts_n 1047 * Maximum number of packets in array. 1048 * 1049 * @return 1050 * Number of packets successfully received (<= pkts_n). 1051 */ 1052 uint16_t 1053 mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) 1054 { 1055 struct mlx5_rxq_data *rxq = dpdk_rxq; 1056 const uint32_t strd_n = 1 << rxq->strd_num_n; 1057 const uint32_t strd_sz = 1 << rxq->strd_sz_n; 1058 const uint32_t cq_mask = (1 << rxq->cqe_n) - 1; 1059 const uint32_t wq_mask = (1 << rxq->elts_n) - 1; 1060 volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; 1061 unsigned int i = 0; 1062 uint32_t rq_ci = rxq->rq_ci; 1063 uint16_t consumed_strd = rxq->consumed_strd; 1064 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; 1065 1066 while (i < pkts_n) { 1067 struct rte_mbuf *pkt; 1068 int ret; 1069 uint32_t len; 1070 uint16_t strd_cnt; 1071 uint16_t strd_idx; 1072 uint32_t byte_cnt; 1073 volatile struct mlx5_mini_cqe8 *mcqe = NULL; 1074 enum mlx5_rqx_code rxq_code; 1075 1076 if (consumed_strd == strd_n) { 1077 /* Replace WQE if the buffer is still in use. */ 1078 mprq_buf_replace(rxq, rq_ci & wq_mask); 1079 /* Advance to the next WQE. */ 1080 consumed_strd = 0; 1081 ++rq_ci; 1082 buf = (*rxq->mprq_bufs)[rq_ci & wq_mask]; 1083 } 1084 cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask]; 1085 ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe); 1086 if (!ret) 1087 break; 1088 byte_cnt = ret; 1089 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT; 1090 MLX5_ASSERT((int)len >= (rxq->crc_present << 2)); 1091 if (rxq->crc_present) 1092 len -= RTE_ETHER_CRC_LEN; 1093 if (mcqe && 1094 rxq->mcqe_format == MLX5_CQE_RESP_FORMAT_FTAG_STRIDX) 1095 strd_cnt = (len / strd_sz) + !!(len % strd_sz); 1096 else 1097 strd_cnt = (byte_cnt & MLX5_MPRQ_STRIDE_NUM_MASK) >> 1098 MLX5_MPRQ_STRIDE_NUM_SHIFT; 1099 MLX5_ASSERT(strd_cnt); 1100 consumed_strd += strd_cnt; 1101 if (byte_cnt & MLX5_MPRQ_FILLER_MASK) 1102 continue; 1103 strd_idx = rte_be_to_cpu_16(mcqe == NULL ? 1104 cqe->wqe_counter : 1105 mcqe->stride_idx); 1106 MLX5_ASSERT(strd_idx < strd_n); 1107 MLX5_ASSERT(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci) & 1108 wq_mask)); 1109 pkt = rte_pktmbuf_alloc(rxq->mp); 1110 if (unlikely(pkt == NULL)) { 1111 ++rxq->stats.rx_nombuf; 1112 break; 1113 } 1114 len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >> MLX5_MPRQ_LEN_SHIFT; 1115 MLX5_ASSERT((int)len >= (rxq->crc_present << 2)); 1116 if (rxq->crc_present) 1117 len -= RTE_ETHER_CRC_LEN; 1118 rxq_code = mprq_buf_to_pkt(rxq, pkt, len, buf, 1119 strd_idx, strd_cnt); 1120 if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) { 1121 rte_pktmbuf_free_seg(pkt); 1122 if (rxq_code == MLX5_RXQ_CODE_DROPPED) { 1123 ++rxq->stats.idropped; 1124 continue; 1125 } 1126 if (rxq_code == MLX5_RXQ_CODE_NOMBUF) { 1127 ++rxq->stats.rx_nombuf; 1128 break; 1129 } 1130 } 1131 rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe); 1132 if (cqe->lro_num_seg > 1) { 1133 mlx5_lro_update_hdr(rte_pktmbuf_mtod(pkt, uint8_t *), 1134 cqe, mcqe, rxq, len); 1135 pkt->ol_flags |= PKT_RX_LRO; 1136 pkt->tso_segsz = len / cqe->lro_num_seg; 1137 } 1138 PKT_LEN(pkt) = len; 1139 PORT(pkt) = rxq->port_id; 1140 #ifdef MLX5_PMD_SOFT_COUNTERS 1141 /* Increment bytes counter. */ 1142 rxq->stats.ibytes += PKT_LEN(pkt); 1143 #endif 1144 /* Return packet. */ 1145 *(pkts++) = pkt; 1146 ++i; 1147 } 1148 /* Update the consumer indexes. */ 1149 rxq->consumed_strd = consumed_strd; 1150 rte_io_wmb(); 1151 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 1152 if (rq_ci != rxq->rq_ci) { 1153 rxq->rq_ci = rq_ci; 1154 rte_io_wmb(); 1155 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 1156 } 1157 #ifdef MLX5_PMD_SOFT_COUNTERS 1158 /* Increment packets counter. */ 1159 rxq->stats.ipackets += i; 1160 #endif 1161 return i; 1162 } 1163 1164 /** 1165 * Dummy DPDK callback for RX. 1166 * 1167 * This function is used to temporarily replace the real callback during 1168 * unsafe control operations on the queue, or in case of error. 1169 * 1170 * @param dpdk_rxq 1171 * Generic pointer to RX queue structure. 1172 * @param[out] pkts 1173 * Array to store received packets. 1174 * @param pkts_n 1175 * Maximum number of packets in array. 1176 * 1177 * @return 1178 * Number of packets successfully received (<= pkts_n). 1179 */ 1180 uint16_t 1181 removed_rx_burst(void *dpdk_rxq __rte_unused, 1182 struct rte_mbuf **pkts __rte_unused, 1183 uint16_t pkts_n __rte_unused) 1184 { 1185 rte_mb(); 1186 return 0; 1187 } 1188 1189 /* 1190 * Vectorized Rx routines are not compiled in when required vector instructions 1191 * are not supported on a target architecture. 1192 * The following null stubs are needed for linkage when those are not included 1193 * outside of this file (e.g. mlx5_rxtx_vec_sse.c for x86). 1194 */ 1195 1196 __rte_weak uint16_t 1197 mlx5_rx_burst_vec(void *dpdk_rxq __rte_unused, 1198 struct rte_mbuf **pkts __rte_unused, 1199 uint16_t pkts_n __rte_unused) 1200 { 1201 return 0; 1202 } 1203 1204 __rte_weak uint16_t 1205 mlx5_rx_burst_mprq_vec(void *dpdk_rxq __rte_unused, 1206 struct rte_mbuf **pkts __rte_unused, 1207 uint16_t pkts_n __rte_unused) 1208 { 1209 return 0; 1210 } 1211 1212 __rte_weak int 1213 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused) 1214 { 1215 return -ENOTSUP; 1216 } 1217 1218 __rte_weak int 1219 mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused) 1220 { 1221 return -ENOTSUP; 1222 } 1223 1224