1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <stddef.h> 7 #include <errno.h> 8 #include <string.h> 9 #include <stdint.h> 10 #include <fcntl.h> 11 #include <sys/queue.h> 12 13 #include <rte_mbuf.h> 14 #include <rte_malloc.h> 15 #include <ethdev_driver.h> 16 #include <rte_common.h> 17 #include <rte_interrupts.h> 18 #include <rte_debug.h> 19 #include <rte_io.h> 20 #include <rte_eal_paging.h> 21 22 #include <mlx5_glue.h> 23 #include <mlx5_malloc.h> 24 #include <mlx5_common.h> 25 #include <mlx5_common_mr.h> 26 27 #include "mlx5_defs.h" 28 #include "mlx5.h" 29 #include "mlx5_rx.h" 30 #include "mlx5_utils.h" 31 #include "mlx5_autoconf.h" 32 #include "mlx5_devx.h" 33 #include "rte_pmd_mlx5.h" 34 35 36 /* Default RSS hash key also used for ConnectX-3. */ 37 uint8_t rss_hash_default_key[] = { 38 0x2c, 0xc6, 0x81, 0xd1, 39 0x5b, 0xdb, 0xf4, 0xf7, 40 0xfc, 0xa2, 0x83, 0x19, 41 0xdb, 0x1a, 0x3e, 0x94, 42 0x6b, 0x9e, 0x38, 0xd9, 43 0x2c, 0x9c, 0x03, 0xd1, 44 0xad, 0x99, 0x44, 0xa7, 45 0xd9, 0x56, 0x3d, 0x59, 46 0x06, 0x3c, 0x25, 0xf3, 47 0xfc, 0x1f, 0xdc, 0x2a, 48 }; 49 50 /* Length of the default RSS hash key. */ 51 static_assert(MLX5_RSS_HASH_KEY_LEN == 52 (unsigned int)sizeof(rss_hash_default_key), 53 "wrong RSS default key size."); 54 55 /** 56 * Calculate the number of CQEs in CQ for the Rx queue. 57 * 58 * @param rxq_data 59 * Pointer to receive queue structure. 60 * 61 * @return 62 * Number of CQEs in CQ. 63 */ 64 unsigned int 65 mlx5_rxq_cqe_num(struct mlx5_rxq_data *rxq_data) 66 { 67 unsigned int cqe_n; 68 unsigned int wqe_n = 1 << rxq_data->elts_n; 69 70 if (mlx5_rxq_mprq_enabled(rxq_data)) 71 cqe_n = wqe_n * RTE_BIT32(rxq_data->log_strd_num) - 1; 72 else 73 cqe_n = wqe_n - 1; 74 return cqe_n; 75 } 76 77 /** 78 * Allocate RX queue elements for Multi-Packet RQ. 79 * 80 * @param rxq_ctrl 81 * Pointer to RX queue structure. 82 * 83 * @return 84 * 0 on success, a negative errno value otherwise and rte_errno is set. 85 */ 86 static int 87 rxq_alloc_elts_mprq(struct mlx5_rxq_ctrl *rxq_ctrl) 88 { 89 struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; 90 unsigned int wqe_n = 1 << rxq->elts_n; 91 unsigned int i; 92 int err; 93 94 /* Iterate on segments. */ 95 for (i = 0; i <= wqe_n; ++i) { 96 struct mlx5_mprq_buf *buf; 97 98 if (rte_mempool_get(rxq->mprq_mp, (void **)&buf) < 0) { 99 DRV_LOG(ERR, "port %u empty mbuf pool", rxq->port_id); 100 rte_errno = ENOMEM; 101 goto error; 102 } 103 if (i < wqe_n) 104 (*rxq->mprq_bufs)[i] = buf; 105 else 106 rxq->mprq_repl = buf; 107 } 108 DRV_LOG(DEBUG, 109 "port %u MPRQ queue %u allocated and configured %u segments", 110 rxq->port_id, rxq->idx, wqe_n); 111 return 0; 112 error: 113 err = rte_errno; /* Save rte_errno before cleanup. */ 114 wqe_n = i; 115 for (i = 0; (i != wqe_n); ++i) { 116 if ((*rxq->mprq_bufs)[i] != NULL) 117 rte_mempool_put(rxq->mprq_mp, 118 (*rxq->mprq_bufs)[i]); 119 (*rxq->mprq_bufs)[i] = NULL; 120 } 121 DRV_LOG(DEBUG, "port %u MPRQ queue %u failed, freed everything", 122 rxq->port_id, rxq->idx); 123 rte_errno = err; /* Restore rte_errno. */ 124 return -rte_errno; 125 } 126 127 /** 128 * Allocate RX queue elements for Single-Packet RQ. 129 * 130 * @param rxq_ctrl 131 * Pointer to RX queue structure. 132 * 133 * @return 134 * 0 on success, negative errno value on failure. 135 */ 136 static int 137 rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl) 138 { 139 const unsigned int sges_n = 1 << rxq_ctrl->rxq.sges_n; 140 unsigned int elts_n = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 141 RTE_BIT32(rxq_ctrl->rxq.elts_n) * 142 RTE_BIT32(rxq_ctrl->rxq.log_strd_num) : 143 RTE_BIT32(rxq_ctrl->rxq.elts_n); 144 bool has_vec_support = mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0; 145 unsigned int i; 146 int err; 147 148 /* Iterate on segments. */ 149 for (i = 0; (i != elts_n); ++i) { 150 struct mlx5_eth_rxseg *seg = &rxq_ctrl->rxq.rxseg[i % sges_n]; 151 struct rte_mbuf *buf; 152 153 buf = rte_pktmbuf_alloc(seg->mp); 154 if (buf == NULL) { 155 if (rxq_ctrl->share_group == 0) 156 DRV_LOG(ERR, "port %u queue %u empty mbuf pool", 157 RXQ_PORT_ID(rxq_ctrl), 158 rxq_ctrl->rxq.idx); 159 else 160 DRV_LOG(ERR, "share group %u queue %u empty mbuf pool", 161 rxq_ctrl->share_group, 162 rxq_ctrl->share_qid); 163 rte_errno = ENOMEM; 164 goto error; 165 } 166 /* Only vectored Rx routines rely on headroom size. */ 167 MLX5_ASSERT(!has_vec_support || 168 DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM); 169 /* Buffer is supposed to be empty. */ 170 MLX5_ASSERT(rte_pktmbuf_data_len(buf) == 0); 171 MLX5_ASSERT(rte_pktmbuf_pkt_len(buf) == 0); 172 MLX5_ASSERT(!buf->next); 173 SET_DATA_OFF(buf, seg->offset); 174 PORT(buf) = rxq_ctrl->rxq.port_id; 175 DATA_LEN(buf) = seg->length; 176 PKT_LEN(buf) = seg->length; 177 NB_SEGS(buf) = 1; 178 (*rxq_ctrl->rxq.elts)[i] = buf; 179 } 180 /* If Rx vector is activated. */ 181 if (has_vec_support) { 182 struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; 183 struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; 184 struct rte_pktmbuf_pool_private *priv = 185 (struct rte_pktmbuf_pool_private *) 186 rte_mempool_get_priv(rxq_ctrl->rxq.mp); 187 int j; 188 189 /* Initialize default rearm_data for vPMD. */ 190 mbuf_init->data_off = RTE_PKTMBUF_HEADROOM; 191 rte_mbuf_refcnt_set(mbuf_init, 1); 192 mbuf_init->nb_segs = 1; 193 /* For shared queues port is provided in CQE */ 194 mbuf_init->port = rxq->shared ? 0 : rxq->port_id; 195 if (priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) 196 mbuf_init->ol_flags = RTE_MBUF_F_EXTERNAL; 197 /* 198 * prevent compiler reordering: 199 * rearm_data covers previous fields. 200 */ 201 rte_compiler_barrier(); 202 rxq->mbuf_initializer = 203 *(rte_xmm_t *)&mbuf_init->rearm_data; 204 /* Padding with a fake mbuf for vectorized Rx. */ 205 for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j) 206 (*rxq->elts)[elts_n + j] = &rxq->fake_mbuf; 207 } 208 if (rxq_ctrl->share_group == 0) 209 DRV_LOG(DEBUG, 210 "port %u SPRQ queue %u allocated and configured %u segments (max %u packets)", 211 RXQ_PORT_ID(rxq_ctrl), rxq_ctrl->rxq.idx, elts_n, 212 elts_n / (1 << rxq_ctrl->rxq.sges_n)); 213 else 214 DRV_LOG(DEBUG, 215 "share group %u SPRQ queue %u allocated and configured %u segments (max %u packets)", 216 rxq_ctrl->share_group, rxq_ctrl->share_qid, elts_n, 217 elts_n / (1 << rxq_ctrl->rxq.sges_n)); 218 return 0; 219 error: 220 err = rte_errno; /* Save rte_errno before cleanup. */ 221 elts_n = i; 222 for (i = 0; (i != elts_n); ++i) { 223 if ((*rxq_ctrl->rxq.elts)[i] != NULL) 224 rte_pktmbuf_free_seg((*rxq_ctrl->rxq.elts)[i]); 225 (*rxq_ctrl->rxq.elts)[i] = NULL; 226 } 227 if (rxq_ctrl->share_group == 0) 228 DRV_LOG(DEBUG, "port %u SPRQ queue %u failed, freed everything", 229 RXQ_PORT_ID(rxq_ctrl), rxq_ctrl->rxq.idx); 230 else 231 DRV_LOG(DEBUG, "share group %u SPRQ queue %u failed, freed everything", 232 rxq_ctrl->share_group, rxq_ctrl->share_qid); 233 rte_errno = err; /* Restore rte_errno. */ 234 return -rte_errno; 235 } 236 237 /** 238 * Allocate RX queue elements. 239 * 240 * @param rxq_ctrl 241 * Pointer to RX queue structure. 242 * 243 * @return 244 * 0 on success, negative errno value on failure. 245 */ 246 int 247 rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) 248 { 249 int ret = 0; 250 251 /** 252 * For MPRQ we need to allocate both MPRQ buffers 253 * for WQEs and simple mbufs for vector processing. 254 */ 255 if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq)) 256 ret = rxq_alloc_elts_mprq(rxq_ctrl); 257 if (ret == 0) 258 ret = rxq_alloc_elts_sprq(rxq_ctrl); 259 return ret; 260 } 261 262 /** 263 * Free RX queue elements for Multi-Packet RQ. 264 * 265 * @param rxq_ctrl 266 * Pointer to RX queue structure. 267 */ 268 static void 269 rxq_free_elts_mprq(struct mlx5_rxq_ctrl *rxq_ctrl) 270 { 271 struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; 272 uint16_t i; 273 274 DRV_LOG(DEBUG, "port %u Multi-Packet Rx queue %u freeing %d WRs", 275 rxq->port_id, rxq->idx, (1u << rxq->elts_n)); 276 if (rxq->mprq_bufs == NULL) 277 return; 278 for (i = 0; (i != (1u << rxq->elts_n)); ++i) { 279 if ((*rxq->mprq_bufs)[i] != NULL) 280 mlx5_mprq_buf_free((*rxq->mprq_bufs)[i]); 281 (*rxq->mprq_bufs)[i] = NULL; 282 } 283 if (rxq->mprq_repl != NULL) { 284 mlx5_mprq_buf_free(rxq->mprq_repl); 285 rxq->mprq_repl = NULL; 286 } 287 } 288 289 /** 290 * Free RX queue elements for Single-Packet RQ. 291 * 292 * @param rxq_ctrl 293 * Pointer to RX queue structure. 294 */ 295 static void 296 rxq_free_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl) 297 { 298 struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; 299 const uint16_t q_n = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 300 RTE_BIT32(rxq->elts_n) * RTE_BIT32(rxq->log_strd_num) : 301 RTE_BIT32(rxq->elts_n); 302 const uint16_t q_mask = q_n - 1; 303 uint16_t elts_ci = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 304 rxq->elts_ci : rxq->rq_ci; 305 uint16_t used = q_n - (elts_ci - rxq->rq_pi); 306 uint16_t i; 307 308 if (rxq_ctrl->share_group == 0) 309 DRV_LOG(DEBUG, "port %u Rx queue %u freeing %d WRs", 310 RXQ_PORT_ID(rxq_ctrl), rxq->idx, q_n); 311 else 312 DRV_LOG(DEBUG, "share group %u Rx queue %u freeing %d WRs", 313 rxq_ctrl->share_group, rxq_ctrl->share_qid, q_n); 314 if (rxq->elts == NULL) 315 return; 316 /** 317 * Some mbuf in the Ring belongs to the application. 318 * They cannot be freed. 319 */ 320 if (mlx5_rxq_check_vec_support(rxq) > 0) { 321 for (i = 0; i < used; ++i) 322 (*rxq->elts)[(elts_ci + i) & q_mask] = NULL; 323 rxq->rq_pi = elts_ci; 324 } 325 for (i = 0; i != q_n; ++i) { 326 if ((*rxq->elts)[i] != NULL) 327 rte_pktmbuf_free_seg((*rxq->elts)[i]); 328 (*rxq->elts)[i] = NULL; 329 } 330 } 331 332 /** 333 * Free RX queue elements. 334 * 335 * @param rxq_ctrl 336 * Pointer to RX queue structure. 337 */ 338 static void 339 rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl) 340 { 341 /* 342 * For MPRQ we need to allocate both MPRQ buffers 343 * for WQEs and simple mbufs for vector processing. 344 */ 345 if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq)) 346 rxq_free_elts_mprq(rxq_ctrl); 347 rxq_free_elts_sprq(rxq_ctrl); 348 } 349 350 /** 351 * Returns the per-queue supported offloads. 352 * 353 * @param dev 354 * Pointer to Ethernet device. 355 * 356 * @return 357 * Supported Rx offloads. 358 */ 359 uint64_t 360 mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev) 361 { 362 struct mlx5_priv *priv = dev->data->dev_private; 363 uint64_t offloads = (RTE_ETH_RX_OFFLOAD_SCATTER | 364 RTE_ETH_RX_OFFLOAD_TIMESTAMP | 365 RTE_ETH_RX_OFFLOAD_RSS_HASH); 366 367 if (!priv->config.mprq.enabled) 368 offloads |= RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT; 369 if (priv->sh->config.hw_fcs_strip) 370 offloads |= RTE_ETH_RX_OFFLOAD_KEEP_CRC; 371 if (priv->sh->dev_cap.hw_csum) 372 offloads |= (RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 373 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 374 RTE_ETH_RX_OFFLOAD_TCP_CKSUM); 375 if (priv->sh->dev_cap.hw_vlan_strip) 376 offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 377 if (priv->sh->dev_cap.lro_supported) 378 offloads |= RTE_ETH_RX_OFFLOAD_TCP_LRO; 379 return offloads; 380 } 381 382 383 /** 384 * Returns the per-port supported offloads. 385 * 386 * @return 387 * Supported Rx offloads. 388 */ 389 uint64_t 390 mlx5_get_rx_port_offloads(void) 391 { 392 uint64_t offloads = RTE_ETH_RX_OFFLOAD_VLAN_FILTER; 393 394 return offloads; 395 } 396 397 /** 398 * Verify if the queue can be released. 399 * 400 * @param dev 401 * Pointer to Ethernet device. 402 * @param idx 403 * RX queue index. 404 * 405 * @return 406 * 1 if the queue can be released 407 * 0 if the queue can not be released, there are references to it. 408 * Negative errno and rte_errno is set if queue doesn't exist. 409 */ 410 static int 411 mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx) 412 { 413 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 414 415 if (rxq == NULL) { 416 rte_errno = EINVAL; 417 return -rte_errno; 418 } 419 return (__atomic_load_n(&rxq->refcnt, __ATOMIC_RELAXED) == 1); 420 } 421 422 /* Fetches and drops all SW-owned and error CQEs to synchronize CQ. */ 423 static void 424 rxq_sync_cq(struct mlx5_rxq_data *rxq) 425 { 426 const uint16_t cqe_n = 1 << rxq->cqe_n; 427 const uint16_t cqe_mask = cqe_n - 1; 428 volatile struct mlx5_cqe *cqe; 429 int ret, i; 430 431 i = cqe_n; 432 do { 433 cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask]; 434 ret = check_cqe(cqe, cqe_n, rxq->cq_ci); 435 if (ret == MLX5_CQE_STATUS_HW_OWN) 436 break; 437 if (ret == MLX5_CQE_STATUS_ERR) { 438 rxq->cq_ci++; 439 continue; 440 } 441 MLX5_ASSERT(ret == MLX5_CQE_STATUS_SW_OWN); 442 if (MLX5_CQE_FORMAT(cqe->op_own) != MLX5_COMPRESSED) { 443 rxq->cq_ci++; 444 continue; 445 } 446 /* Compute the next non compressed CQE. */ 447 rxq->cq_ci += rte_be_to_cpu_32(cqe->byte_cnt); 448 449 } while (--i); 450 /* Move all CQEs to HW ownership, including possible MiniCQEs. */ 451 for (i = 0; i < cqe_n; i++) { 452 cqe = &(*rxq->cqes)[i]; 453 cqe->op_own = MLX5_CQE_INVALIDATE; 454 } 455 /* Resync CQE and WQE (WQ in RESET state). */ 456 rte_io_wmb(); 457 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci); 458 rte_io_wmb(); 459 *rxq->rq_db = rte_cpu_to_be_32(0); 460 rte_io_wmb(); 461 } 462 463 /** 464 * Rx queue stop. Device queue goes to the RESET state, 465 * all involved mbufs are freed from WQ. 466 * 467 * @param dev 468 * Pointer to Ethernet device structure. 469 * @param idx 470 * RX queue index. 471 * 472 * @return 473 * 0 on success, a negative errno value otherwise and rte_errno is set. 474 */ 475 int 476 mlx5_rx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t idx) 477 { 478 struct mlx5_priv *priv = dev->data->dev_private; 479 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 480 struct mlx5_rxq_ctrl *rxq_ctrl = rxq->ctrl; 481 int ret; 482 483 MLX5_ASSERT(rxq != NULL && rxq_ctrl != NULL); 484 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 485 ret = priv->obj_ops.rxq_obj_modify(rxq, MLX5_RXQ_MOD_RDY2RST); 486 if (ret) { 487 DRV_LOG(ERR, "Cannot change Rx WQ state to RESET: %s", 488 strerror(errno)); 489 rte_errno = errno; 490 return ret; 491 } 492 /* Remove all processes CQEs. */ 493 rxq_sync_cq(&rxq_ctrl->rxq); 494 /* Free all involved mbufs. */ 495 rxq_free_elts(rxq_ctrl); 496 /* Set the actual queue state. */ 497 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STOPPED; 498 return 0; 499 } 500 501 /** 502 * Rx queue stop. Device queue goes to the RESET state, 503 * all involved mbufs are freed from WQ. 504 * 505 * @param dev 506 * Pointer to Ethernet device structure. 507 * @param idx 508 * RX queue index. 509 * 510 * @return 511 * 0 on success, a negative errno value otherwise and rte_errno is set. 512 */ 513 int 514 mlx5_rx_queue_stop(struct rte_eth_dev *dev, uint16_t idx) 515 { 516 eth_rx_burst_t pkt_burst = dev->rx_pkt_burst; 517 int ret; 518 519 if (rte_eth_dev_is_rx_hairpin_queue(dev, idx)) { 520 DRV_LOG(ERR, "Hairpin queue can't be stopped"); 521 rte_errno = EINVAL; 522 return -EINVAL; 523 } 524 if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STOPPED) 525 return 0; 526 /* 527 * Vectorized Rx burst requires the CQ and RQ indices 528 * synchronized, that might be broken on RQ restart 529 * and cause Rx malfunction, so queue stopping is 530 * not supported if vectorized Rx burst is engaged. 531 * The routine pointer depends on the process 532 * type, should perform check there. 533 */ 534 if (pkt_burst == mlx5_rx_burst_vec) { 535 DRV_LOG(ERR, "Rx queue stop is not supported " 536 "for vectorized Rx"); 537 rte_errno = EINVAL; 538 return -EINVAL; 539 } 540 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 541 ret = mlx5_mp_os_req_queue_control(dev, idx, 542 MLX5_MP_REQ_QUEUE_RX_STOP); 543 } else { 544 ret = mlx5_rx_queue_stop_primary(dev, idx); 545 } 546 return ret; 547 } 548 549 /** 550 * Rx queue start. Device queue goes to the ready state, 551 * all required mbufs are allocated and WQ is replenished. 552 * 553 * @param dev 554 * Pointer to Ethernet device structure. 555 * @param idx 556 * RX queue index. 557 * 558 * @return 559 * 0 on success, a negative errno value otherwise and rte_errno is set. 560 */ 561 int 562 mlx5_rx_queue_start_primary(struct rte_eth_dev *dev, uint16_t idx) 563 { 564 struct mlx5_priv *priv = dev->data->dev_private; 565 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 566 struct mlx5_rxq_data *rxq_data = &rxq->ctrl->rxq; 567 int ret; 568 569 MLX5_ASSERT(rxq != NULL && rxq->ctrl != NULL); 570 MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); 571 /* Allocate needed buffers. */ 572 ret = rxq_alloc_elts(rxq->ctrl); 573 if (ret) { 574 DRV_LOG(ERR, "Cannot reallocate buffers for Rx WQ"); 575 rte_errno = errno; 576 return ret; 577 } 578 rte_io_wmb(); 579 *rxq_data->cq_db = rte_cpu_to_be_32(rxq_data->cq_ci); 580 rte_io_wmb(); 581 /* Reset RQ consumer before moving queue to READY state. */ 582 *rxq_data->rq_db = rte_cpu_to_be_32(0); 583 rte_io_wmb(); 584 ret = priv->obj_ops.rxq_obj_modify(rxq, MLX5_RXQ_MOD_RST2RDY); 585 if (ret) { 586 DRV_LOG(ERR, "Cannot change Rx WQ state to READY: %s", 587 strerror(errno)); 588 rte_errno = errno; 589 return ret; 590 } 591 /* Reinitialize RQ - set WQEs. */ 592 mlx5_rxq_initialize(rxq_data); 593 rxq_data->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR; 594 /* Set actual queue state. */ 595 dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED; 596 return 0; 597 } 598 599 /** 600 * Rx queue start. Device queue goes to the ready state, 601 * all required mbufs are allocated and WQ is replenished. 602 * 603 * @param dev 604 * Pointer to Ethernet device structure. 605 * @param idx 606 * RX queue index. 607 * 608 * @return 609 * 0 on success, a negative errno value otherwise and rte_errno is set. 610 */ 611 int 612 mlx5_rx_queue_start(struct rte_eth_dev *dev, uint16_t idx) 613 { 614 int ret; 615 616 if (rte_eth_dev_is_rx_hairpin_queue(dev, idx)) { 617 DRV_LOG(ERR, "Hairpin queue can't be started"); 618 rte_errno = EINVAL; 619 return -EINVAL; 620 } 621 if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STARTED) 622 return 0; 623 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 624 ret = mlx5_mp_os_req_queue_control(dev, idx, 625 MLX5_MP_REQ_QUEUE_RX_START); 626 } else { 627 ret = mlx5_rx_queue_start_primary(dev, idx); 628 } 629 return ret; 630 } 631 632 /** 633 * Rx queue presetup checks. 634 * 635 * @param dev 636 * Pointer to Ethernet device structure. 637 * @param idx 638 * RX queue index. 639 * @param desc 640 * Number of descriptors to configure in queue. 641 * @param[out] rxq_ctrl 642 * Address of pointer to shared Rx queue control. 643 * 644 * @return 645 * 0 on success, a negative errno value otherwise and rte_errno is set. 646 */ 647 static int 648 mlx5_rx_queue_pre_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t *desc, 649 struct mlx5_rxq_ctrl **rxq_ctrl) 650 { 651 struct mlx5_priv *priv = dev->data->dev_private; 652 struct mlx5_rxq_priv *rxq; 653 bool empty; 654 655 if (!rte_is_power_of_2(*desc)) { 656 *desc = 1 << log2above(*desc); 657 DRV_LOG(WARNING, 658 "port %u increased number of descriptors in Rx queue %u" 659 " to the next power of two (%d)", 660 dev->data->port_id, idx, *desc); 661 } 662 DRV_LOG(DEBUG, "port %u configuring Rx queue %u for %u descriptors", 663 dev->data->port_id, idx, *desc); 664 if (idx >= priv->rxqs_n) { 665 DRV_LOG(ERR, "port %u Rx queue index out of range (%u >= %u)", 666 dev->data->port_id, idx, priv->rxqs_n); 667 rte_errno = EOVERFLOW; 668 return -rte_errno; 669 } 670 if (rxq_ctrl == NULL || *rxq_ctrl == NULL) 671 return 0; 672 if (!(*rxq_ctrl)->rxq.shared) { 673 if (!mlx5_rxq_releasable(dev, idx)) { 674 DRV_LOG(ERR, "port %u unable to release queue index %u", 675 dev->data->port_id, idx); 676 rte_errno = EBUSY; 677 return -rte_errno; 678 } 679 mlx5_rxq_release(dev, idx); 680 } else { 681 if ((*rxq_ctrl)->obj != NULL) 682 /* Some port using shared Rx queue has been started. */ 683 return 0; 684 /* Release all owner RxQ to reconfigure Shared RxQ. */ 685 do { 686 rxq = LIST_FIRST(&(*rxq_ctrl)->owners); 687 LIST_REMOVE(rxq, owner_entry); 688 empty = LIST_EMPTY(&(*rxq_ctrl)->owners); 689 mlx5_rxq_release(ETH_DEV(rxq->priv), rxq->idx); 690 } while (!empty); 691 *rxq_ctrl = NULL; 692 } 693 return 0; 694 } 695 696 /** 697 * Get the shared Rx queue object that matches group and queue index. 698 * 699 * @param dev 700 * Pointer to Ethernet device structure. 701 * @param group 702 * Shared RXQ group. 703 * @param share_qid 704 * Shared RX queue index. 705 * 706 * @return 707 * Shared RXQ object that matching, or NULL if not found. 708 */ 709 static struct mlx5_rxq_ctrl * 710 mlx5_shared_rxq_get(struct rte_eth_dev *dev, uint32_t group, uint16_t share_qid) 711 { 712 struct mlx5_rxq_ctrl *rxq_ctrl; 713 struct mlx5_priv *priv = dev->data->dev_private; 714 715 LIST_FOREACH(rxq_ctrl, &priv->sh->shared_rxqs, share_entry) { 716 if (rxq_ctrl->share_group == group && 717 rxq_ctrl->share_qid == share_qid) 718 return rxq_ctrl; 719 } 720 return NULL; 721 } 722 723 /** 724 * Check whether requested Rx queue configuration matches shared RXQ. 725 * 726 * @param rxq_ctrl 727 * Pointer to shared RXQ. 728 * @param dev 729 * Pointer to Ethernet device structure. 730 * @param idx 731 * Queue index. 732 * @param desc 733 * Number of descriptors to configure in queue. 734 * @param socket 735 * NUMA socket on which memory must be allocated. 736 * @param[in] conf 737 * Thresholds parameters. 738 * @param mp 739 * Memory pool for buffer allocations. 740 * 741 * @return 742 * 0 on success, a negative errno value otherwise and rte_errno is set. 743 */ 744 static bool 745 mlx5_shared_rxq_match(struct mlx5_rxq_ctrl *rxq_ctrl, struct rte_eth_dev *dev, 746 uint16_t idx, uint16_t desc, unsigned int socket, 747 const struct rte_eth_rxconf *conf, 748 struct rte_mempool *mp) 749 { 750 struct mlx5_priv *spriv = LIST_FIRST(&rxq_ctrl->owners)->priv; 751 struct mlx5_priv *priv = dev->data->dev_private; 752 unsigned int i; 753 754 RTE_SET_USED(conf); 755 if (rxq_ctrl->socket != socket) { 756 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: socket mismatch", 757 dev->data->port_id, idx); 758 return false; 759 } 760 if (rxq_ctrl->rxq.elts_n != log2above(desc)) { 761 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: descriptor number mismatch", 762 dev->data->port_id, idx); 763 return false; 764 } 765 if (priv->mtu != spriv->mtu) { 766 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: mtu mismatch", 767 dev->data->port_id, idx); 768 return false; 769 } 770 if (priv->dev_data->dev_conf.intr_conf.rxq != 771 spriv->dev_data->dev_conf.intr_conf.rxq) { 772 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: interrupt mismatch", 773 dev->data->port_id, idx); 774 return false; 775 } 776 if (mp != NULL && rxq_ctrl->rxq.mp != mp) { 777 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: mempool mismatch", 778 dev->data->port_id, idx); 779 return false; 780 } else if (mp == NULL) { 781 if (conf->rx_nseg != rxq_ctrl->rxseg_n) { 782 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: segment number mismatch", 783 dev->data->port_id, idx); 784 return false; 785 } 786 for (i = 0; i < conf->rx_nseg; i++) { 787 if (memcmp(&conf->rx_seg[i].split, &rxq_ctrl->rxseg[i], 788 sizeof(struct rte_eth_rxseg_split))) { 789 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: segment %u configuration mismatch", 790 dev->data->port_id, idx, i); 791 return false; 792 } 793 } 794 } 795 if (priv->config.hw_padding != spriv->config.hw_padding) { 796 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: padding mismatch", 797 dev->data->port_id, idx); 798 return false; 799 } 800 if (priv->config.cqe_comp != spriv->config.cqe_comp || 801 (priv->config.cqe_comp && 802 priv->config.cqe_comp_fmt != spriv->config.cqe_comp_fmt)) { 803 DRV_LOG(ERR, "port %u queue index %u failed to join shared group: CQE compression mismatch", 804 dev->data->port_id, idx); 805 return false; 806 } 807 return true; 808 } 809 810 /** 811 * 812 * @param dev 813 * Pointer to Ethernet device structure. 814 * @param idx 815 * RX queue index. 816 * @param desc 817 * Number of descriptors to configure in queue. 818 * @param socket 819 * NUMA socket on which memory must be allocated. 820 * @param[in] conf 821 * Thresholds parameters. 822 * @param mp 823 * Memory pool for buffer allocations. 824 * 825 * @return 826 * 0 on success, a negative errno value otherwise and rte_errno is set. 827 */ 828 int 829 mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 830 unsigned int socket, const struct rte_eth_rxconf *conf, 831 struct rte_mempool *mp) 832 { 833 struct mlx5_priv *priv = dev->data->dev_private; 834 struct mlx5_rxq_priv *rxq; 835 struct mlx5_rxq_ctrl *rxq_ctrl = NULL; 836 struct rte_eth_rxseg_split *rx_seg = 837 (struct rte_eth_rxseg_split *)conf->rx_seg; 838 struct rte_eth_rxseg_split rx_single = {.mp = mp}; 839 uint16_t n_seg = conf->rx_nseg; 840 int res; 841 uint64_t offloads = conf->offloads | 842 dev->data->dev_conf.rxmode.offloads; 843 bool is_extmem = false; 844 845 if ((offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) && 846 !priv->sh->dev_cap.lro_supported) { 847 DRV_LOG(ERR, 848 "Port %u queue %u LRO is configured but not supported.", 849 dev->data->port_id, idx); 850 rte_errno = EINVAL; 851 return -rte_errno; 852 } 853 if (mp) { 854 /* 855 * The parameters should be checked on rte_eth_dev layer. 856 * If mp is specified it means the compatible configuration 857 * without buffer split feature tuning. 858 */ 859 rx_seg = &rx_single; 860 n_seg = 1; 861 is_extmem = rte_pktmbuf_priv_flags(mp) & 862 RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF; 863 } 864 if (n_seg > 1) { 865 /* The offloads should be checked on rte_eth_dev layer. */ 866 MLX5_ASSERT(offloads & RTE_ETH_RX_OFFLOAD_SCATTER); 867 if (!(offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) { 868 DRV_LOG(ERR, "port %u queue index %u split " 869 "offload not configured", 870 dev->data->port_id, idx); 871 rte_errno = ENOSPC; 872 return -rte_errno; 873 } 874 MLX5_ASSERT(n_seg < MLX5_MAX_RXQ_NSEG); 875 } 876 if (conf->share_group > 0) { 877 if (!priv->sh->cdev->config.hca_attr.mem_rq_rmp) { 878 DRV_LOG(ERR, "port %u queue index %u shared Rx queue not supported by fw", 879 dev->data->port_id, idx); 880 rte_errno = EINVAL; 881 return -rte_errno; 882 } 883 if (priv->obj_ops.rxq_obj_new != devx_obj_ops.rxq_obj_new) { 884 DRV_LOG(ERR, "port %u queue index %u shared Rx queue needs DevX api", 885 dev->data->port_id, idx); 886 rte_errno = EINVAL; 887 return -rte_errno; 888 } 889 if (conf->share_qid >= priv->rxqs_n) { 890 DRV_LOG(ERR, "port %u shared Rx queue index %u > number of Rx queues %u", 891 dev->data->port_id, conf->share_qid, 892 priv->rxqs_n); 893 rte_errno = EINVAL; 894 return -rte_errno; 895 } 896 if (priv->config.mprq.enabled) { 897 DRV_LOG(ERR, "port %u shared Rx queue index %u: not supported when MPRQ enabled", 898 dev->data->port_id, conf->share_qid); 899 rte_errno = EINVAL; 900 return -rte_errno; 901 } 902 /* Try to reuse shared RXQ. */ 903 rxq_ctrl = mlx5_shared_rxq_get(dev, conf->share_group, 904 conf->share_qid); 905 if (rxq_ctrl != NULL && 906 !mlx5_shared_rxq_match(rxq_ctrl, dev, idx, desc, socket, 907 conf, mp)) { 908 rte_errno = EINVAL; 909 return -rte_errno; 910 } 911 } 912 res = mlx5_rx_queue_pre_setup(dev, idx, &desc, &rxq_ctrl); 913 if (res) 914 return res; 915 /* Allocate RXQ. */ 916 rxq = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*rxq), 0, 917 SOCKET_ID_ANY); 918 if (!rxq) { 919 DRV_LOG(ERR, "port %u unable to allocate rx queue index %u private data", 920 dev->data->port_id, idx); 921 rte_errno = ENOMEM; 922 return -rte_errno; 923 } 924 if (rxq_ctrl == NULL) { 925 rxq_ctrl = mlx5_rxq_new(dev, idx, desc, socket, conf, rx_seg, 926 n_seg, is_extmem); 927 if (rxq_ctrl == NULL) { 928 DRV_LOG(ERR, "port %u unable to allocate rx queue index %u", 929 dev->data->port_id, idx); 930 mlx5_free(rxq); 931 rte_errno = ENOMEM; 932 return -rte_errno; 933 } 934 } 935 rxq->priv = priv; 936 rxq->idx = idx; 937 (*priv->rxq_privs)[idx] = rxq; 938 /* Join owner list. */ 939 LIST_INSERT_HEAD(&rxq_ctrl->owners, rxq, owner_entry); 940 rxq->ctrl = rxq_ctrl; 941 mlx5_rxq_ref(dev, idx); 942 DRV_LOG(DEBUG, "port %u adding Rx queue %u to list", 943 dev->data->port_id, idx); 944 dev->data->rx_queues[idx] = &rxq_ctrl->rxq; 945 return 0; 946 } 947 948 /** 949 * 950 * @param dev 951 * Pointer to Ethernet device structure. 952 * @param idx 953 * RX queue index. 954 * @param desc 955 * Number of descriptors to configure in queue. 956 * @param hairpin_conf 957 * Hairpin configuration parameters. 958 * 959 * @return 960 * 0 on success, a negative errno value otherwise and rte_errno is set. 961 */ 962 int 963 mlx5_rx_hairpin_queue_setup(struct rte_eth_dev *dev, uint16_t idx, 964 uint16_t desc, 965 const struct rte_eth_hairpin_conf *hairpin_conf) 966 { 967 struct mlx5_priv *priv = dev->data->dev_private; 968 struct mlx5_rxq_priv *rxq; 969 struct mlx5_rxq_ctrl *rxq_ctrl; 970 int res; 971 972 res = mlx5_rx_queue_pre_setup(dev, idx, &desc, NULL); 973 if (res) 974 return res; 975 if (hairpin_conf->peer_count != 1) { 976 rte_errno = EINVAL; 977 DRV_LOG(ERR, "port %u unable to setup Rx hairpin queue index %u" 978 " peer count is %u", dev->data->port_id, 979 idx, hairpin_conf->peer_count); 980 return -rte_errno; 981 } 982 if (hairpin_conf->peers[0].port == dev->data->port_id) { 983 if (hairpin_conf->peers[0].queue >= priv->txqs_n) { 984 rte_errno = EINVAL; 985 DRV_LOG(ERR, "port %u unable to setup Rx hairpin queue" 986 " index %u, Tx %u is larger than %u", 987 dev->data->port_id, idx, 988 hairpin_conf->peers[0].queue, priv->txqs_n); 989 return -rte_errno; 990 } 991 } else { 992 if (hairpin_conf->manual_bind == 0 || 993 hairpin_conf->tx_explicit == 0) { 994 rte_errno = EINVAL; 995 DRV_LOG(ERR, "port %u unable to setup Rx hairpin queue" 996 " index %u peer port %u with attributes %u %u", 997 dev->data->port_id, idx, 998 hairpin_conf->peers[0].port, 999 hairpin_conf->manual_bind, 1000 hairpin_conf->tx_explicit); 1001 return -rte_errno; 1002 } 1003 } 1004 rxq = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*rxq), 0, 1005 SOCKET_ID_ANY); 1006 if (!rxq) { 1007 DRV_LOG(ERR, "port %u unable to allocate hairpin rx queue index %u private data", 1008 dev->data->port_id, idx); 1009 rte_errno = ENOMEM; 1010 return -rte_errno; 1011 } 1012 rxq->priv = priv; 1013 rxq->idx = idx; 1014 (*priv->rxq_privs)[idx] = rxq; 1015 rxq_ctrl = mlx5_rxq_hairpin_new(dev, rxq, desc, hairpin_conf); 1016 if (!rxq_ctrl) { 1017 DRV_LOG(ERR, "port %u unable to allocate hairpin queue index %u", 1018 dev->data->port_id, idx); 1019 mlx5_free(rxq); 1020 (*priv->rxq_privs)[idx] = NULL; 1021 rte_errno = ENOMEM; 1022 return -rte_errno; 1023 } 1024 DRV_LOG(DEBUG, "port %u adding hairpin Rx queue %u to list", 1025 dev->data->port_id, idx); 1026 dev->data->rx_queues[idx] = &rxq_ctrl->rxq; 1027 return 0; 1028 } 1029 1030 /** 1031 * DPDK callback to release a RX queue. 1032 * 1033 * @param dev 1034 * Pointer to Ethernet device structure. 1035 * @param qid 1036 * Receive queue index. 1037 */ 1038 void 1039 mlx5_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1040 { 1041 struct mlx5_rxq_data *rxq = dev->data->rx_queues[qid]; 1042 1043 if (rxq == NULL) 1044 return; 1045 if (!mlx5_rxq_releasable(dev, qid)) 1046 rte_panic("port %u Rx queue %u is still used by a flow and" 1047 " cannot be removed\n", dev->data->port_id, qid); 1048 mlx5_rxq_release(dev, qid); 1049 } 1050 1051 /** 1052 * Allocate queue vector and fill epoll fd list for Rx interrupts. 1053 * 1054 * @param dev 1055 * Pointer to Ethernet device. 1056 * 1057 * @return 1058 * 0 on success, a negative errno value otherwise and rte_errno is set. 1059 */ 1060 int 1061 mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev) 1062 { 1063 struct mlx5_priv *priv = dev->data->dev_private; 1064 unsigned int i; 1065 unsigned int rxqs_n = priv->rxqs_n; 1066 unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); 1067 unsigned int count = 0; 1068 struct rte_intr_handle *intr_handle = dev->intr_handle; 1069 1070 if (!dev->data->dev_conf.intr_conf.rxq) 1071 return 0; 1072 mlx5_rx_intr_vec_disable(dev); 1073 if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) { 1074 DRV_LOG(ERR, 1075 "port %u failed to allocate memory for interrupt" 1076 " vector, Rx interrupts will not be supported", 1077 dev->data->port_id); 1078 rte_errno = ENOMEM; 1079 return -rte_errno; 1080 } 1081 1082 if (rte_intr_type_set(intr_handle, RTE_INTR_HANDLE_EXT)) 1083 return -rte_errno; 1084 1085 for (i = 0; i != n; ++i) { 1086 /* This rxq obj must not be released in this function. */ 1087 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1088 struct mlx5_rxq_obj *rxq_obj = rxq ? rxq->ctrl->obj : NULL; 1089 int rc; 1090 1091 /* Skip queues that cannot request interrupts. */ 1092 if (!rxq_obj || (!rxq_obj->ibv_channel && 1093 !rxq_obj->devx_channel)) { 1094 /* Use invalid intr_vec[] index to disable entry. */ 1095 if (rte_intr_vec_list_index_set(intr_handle, i, 1096 RTE_INTR_VEC_RXTX_OFFSET + RTE_MAX_RXTX_INTR_VEC_ID)) 1097 return -rte_errno; 1098 continue; 1099 } 1100 mlx5_rxq_ref(dev, i); 1101 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { 1102 DRV_LOG(ERR, 1103 "port %u too many Rx queues for interrupt" 1104 " vector size (%d), Rx interrupts cannot be" 1105 " enabled", 1106 dev->data->port_id, RTE_MAX_RXTX_INTR_VEC_ID); 1107 mlx5_rx_intr_vec_disable(dev); 1108 rte_errno = ENOMEM; 1109 return -rte_errno; 1110 } 1111 rc = mlx5_os_set_nonblock_channel_fd(rxq_obj->fd); 1112 if (rc < 0) { 1113 rte_errno = errno; 1114 DRV_LOG(ERR, 1115 "port %u failed to make Rx interrupt file" 1116 " descriptor %d non-blocking for queue index" 1117 " %d", 1118 dev->data->port_id, rxq_obj->fd, i); 1119 mlx5_rx_intr_vec_disable(dev); 1120 return -rte_errno; 1121 } 1122 1123 if (rte_intr_vec_list_index_set(intr_handle, i, 1124 RTE_INTR_VEC_RXTX_OFFSET + count)) 1125 return -rte_errno; 1126 if (rte_intr_efds_index_set(intr_handle, count, 1127 rxq_obj->fd)) 1128 return -rte_errno; 1129 count++; 1130 } 1131 if (!count) 1132 mlx5_rx_intr_vec_disable(dev); 1133 else if (rte_intr_nb_efd_set(intr_handle, count)) 1134 return -rte_errno; 1135 return 0; 1136 } 1137 1138 /** 1139 * Clean up Rx interrupts handler. 1140 * 1141 * @param dev 1142 * Pointer to Ethernet device. 1143 */ 1144 void 1145 mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev) 1146 { 1147 struct mlx5_priv *priv = dev->data->dev_private; 1148 struct rte_intr_handle *intr_handle = dev->intr_handle; 1149 unsigned int i; 1150 unsigned int rxqs_n = priv->rxqs_n; 1151 unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); 1152 1153 if (!dev->data->dev_conf.intr_conf.rxq) 1154 return; 1155 if (rte_intr_vec_list_index_get(intr_handle, 0) < 0) 1156 goto free; 1157 for (i = 0; i != n; ++i) { 1158 if (rte_intr_vec_list_index_get(intr_handle, i) == 1159 RTE_INTR_VEC_RXTX_OFFSET + RTE_MAX_RXTX_INTR_VEC_ID) 1160 continue; 1161 /** 1162 * Need to access directly the queue to release the reference 1163 * kept in mlx5_rx_intr_vec_enable(). 1164 */ 1165 mlx5_rxq_deref(dev, i); 1166 } 1167 free: 1168 rte_intr_free_epoll_fd(intr_handle); 1169 1170 rte_intr_vec_list_free(intr_handle); 1171 1172 rte_intr_nb_efd_set(intr_handle, 0); 1173 } 1174 1175 /** 1176 * MLX5 CQ notification . 1177 * 1178 * @param rxq 1179 * Pointer to receive queue structure. 1180 * @param sq_n_rxq 1181 * Sequence number per receive queue . 1182 */ 1183 static inline void 1184 mlx5_arm_cq(struct mlx5_rxq_data *rxq, int sq_n_rxq) 1185 { 1186 int sq_n = 0; 1187 uint32_t doorbell_hi; 1188 uint64_t doorbell; 1189 1190 sq_n = sq_n_rxq & MLX5_CQ_SQN_MASK; 1191 doorbell_hi = sq_n << MLX5_CQ_SQN_OFFSET | (rxq->cq_ci & MLX5_CI_MASK); 1192 doorbell = (uint64_t)doorbell_hi << 32; 1193 doorbell |= rxq->cqn; 1194 mlx5_doorbell_ring(&rxq->uar_data, rte_cpu_to_be_64(doorbell), 1195 doorbell_hi, &rxq->cq_db[MLX5_CQ_ARM_DB], 0); 1196 } 1197 1198 /** 1199 * DPDK callback for Rx queue interrupt enable. 1200 * 1201 * @param dev 1202 * Pointer to Ethernet device structure. 1203 * @param rx_queue_id 1204 * Rx queue number. 1205 * 1206 * @return 1207 * 0 on success, a negative errno value otherwise and rte_errno is set. 1208 */ 1209 int 1210 mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1211 { 1212 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, rx_queue_id); 1213 if (!rxq) 1214 goto error; 1215 if (rxq->ctrl->irq) { 1216 if (!rxq->ctrl->obj) 1217 goto error; 1218 mlx5_arm_cq(&rxq->ctrl->rxq, rxq->ctrl->rxq.cq_arm_sn); 1219 } 1220 return 0; 1221 error: 1222 rte_errno = EINVAL; 1223 return -rte_errno; 1224 } 1225 1226 /** 1227 * DPDK callback for Rx queue interrupt disable. 1228 * 1229 * @param dev 1230 * Pointer to Ethernet device structure. 1231 * @param rx_queue_id 1232 * Rx queue number. 1233 * 1234 * @return 1235 * 0 on success, a negative errno value otherwise and rte_errno is set. 1236 */ 1237 int 1238 mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1239 { 1240 struct mlx5_priv *priv = dev->data->dev_private; 1241 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, rx_queue_id); 1242 int ret = 0; 1243 1244 if (!rxq) { 1245 rte_errno = EINVAL; 1246 return -rte_errno; 1247 } 1248 if (!rxq->ctrl->obj) 1249 goto error; 1250 if (rxq->ctrl->irq) { 1251 ret = priv->obj_ops.rxq_event_get(rxq->ctrl->obj); 1252 if (ret < 0) 1253 goto error; 1254 rxq->ctrl->rxq.cq_arm_sn++; 1255 } 1256 return 0; 1257 error: 1258 /** 1259 * The ret variable may be EAGAIN which means the get_event function was 1260 * called before receiving one. 1261 */ 1262 if (ret < 0) 1263 rte_errno = errno; 1264 else 1265 rte_errno = EINVAL; 1266 if (rte_errno != EAGAIN) 1267 DRV_LOG(WARNING, "port %u unable to disable interrupt on Rx queue %d", 1268 dev->data->port_id, rx_queue_id); 1269 return -rte_errno; 1270 } 1271 1272 /** 1273 * Verify the Rx queue objects list is empty 1274 * 1275 * @param dev 1276 * Pointer to Ethernet device. 1277 * 1278 * @return 1279 * The number of objects not released. 1280 */ 1281 int 1282 mlx5_rxq_obj_verify(struct rte_eth_dev *dev) 1283 { 1284 struct mlx5_priv *priv = dev->data->dev_private; 1285 int ret = 0; 1286 struct mlx5_rxq_obj *rxq_obj; 1287 1288 LIST_FOREACH(rxq_obj, &priv->rxqsobj, next) { 1289 if (rxq_obj->rxq_ctrl == NULL) 1290 continue; 1291 if (rxq_obj->rxq_ctrl->rxq.shared && 1292 !LIST_EMPTY(&rxq_obj->rxq_ctrl->owners)) 1293 continue; 1294 DRV_LOG(DEBUG, "port %u Rx queue %u still referenced", 1295 dev->data->port_id, rxq_obj->rxq_ctrl->rxq.idx); 1296 ++ret; 1297 } 1298 return ret; 1299 } 1300 1301 /** 1302 * Callback function to initialize mbufs for Multi-Packet RQ. 1303 */ 1304 static inline void 1305 mlx5_mprq_buf_init(struct rte_mempool *mp, void *opaque_arg, 1306 void *_m, unsigned int i __rte_unused) 1307 { 1308 struct mlx5_mprq_buf *buf = _m; 1309 struct rte_mbuf_ext_shared_info *shinfo; 1310 unsigned int strd_n = (unsigned int)(uintptr_t)opaque_arg; 1311 unsigned int j; 1312 1313 memset(_m, 0, sizeof(*buf)); 1314 buf->mp = mp; 1315 __atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED); 1316 for (j = 0; j != strd_n; ++j) { 1317 shinfo = &buf->shinfos[j]; 1318 shinfo->free_cb = mlx5_mprq_buf_free_cb; 1319 shinfo->fcb_opaque = buf; 1320 } 1321 } 1322 1323 /** 1324 * Free mempool of Multi-Packet RQ. 1325 * 1326 * @param dev 1327 * Pointer to Ethernet device. 1328 * 1329 * @return 1330 * 0 on success, negative errno value on failure. 1331 */ 1332 int 1333 mlx5_mprq_free_mp(struct rte_eth_dev *dev) 1334 { 1335 struct mlx5_priv *priv = dev->data->dev_private; 1336 struct rte_mempool *mp = priv->mprq_mp; 1337 unsigned int i; 1338 1339 if (mp == NULL) 1340 return 0; 1341 DRV_LOG(DEBUG, "port %u freeing mempool (%s) for Multi-Packet RQ", 1342 dev->data->port_id, mp->name); 1343 /* 1344 * If a buffer in the pool has been externally attached to a mbuf and it 1345 * is still in use by application, destroying the Rx queue can spoil 1346 * the packet. It is unlikely to happen but if application dynamically 1347 * creates and destroys with holding Rx packets, this can happen. 1348 * 1349 * TODO: It is unavoidable for now because the mempool for Multi-Packet 1350 * RQ isn't provided by application but managed by PMD. 1351 */ 1352 if (!rte_mempool_full(mp)) { 1353 DRV_LOG(ERR, 1354 "port %u mempool for Multi-Packet RQ is still in use", 1355 dev->data->port_id); 1356 rte_errno = EBUSY; 1357 return -rte_errno; 1358 } 1359 rte_mempool_free(mp); 1360 /* Unset mempool for each Rx queue. */ 1361 for (i = 0; i != priv->rxqs_n; ++i) { 1362 struct mlx5_rxq_data *rxq = mlx5_rxq_data_get(dev, i); 1363 1364 if (rxq == NULL) 1365 continue; 1366 rxq->mprq_mp = NULL; 1367 } 1368 priv->mprq_mp = NULL; 1369 return 0; 1370 } 1371 1372 /** 1373 * Allocate a mempool for Multi-Packet RQ. All configured Rx queues share the 1374 * mempool. If already allocated, reuse it if there're enough elements. 1375 * Otherwise, resize it. 1376 * 1377 * @param dev 1378 * Pointer to Ethernet device. 1379 * 1380 * @return 1381 * 0 on success, negative errno value on failure. 1382 */ 1383 int 1384 mlx5_mprq_alloc_mp(struct rte_eth_dev *dev) 1385 { 1386 struct mlx5_priv *priv = dev->data->dev_private; 1387 struct rte_mempool *mp = priv->mprq_mp; 1388 char name[RTE_MEMPOOL_NAMESIZE]; 1389 unsigned int desc = 0; 1390 unsigned int buf_len; 1391 unsigned int obj_num; 1392 unsigned int obj_size; 1393 unsigned int log_strd_num = 0; 1394 unsigned int log_strd_sz = 0; 1395 unsigned int i; 1396 unsigned int n_ibv = 0; 1397 int ret; 1398 1399 if (!mlx5_mprq_enabled(dev)) 1400 return 0; 1401 /* Count the total number of descriptors configured. */ 1402 for (i = 0; i != priv->rxqs_n; ++i) { 1403 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 1404 struct mlx5_rxq_data *rxq; 1405 1406 if (rxq_ctrl == NULL || rxq_ctrl->is_hairpin) 1407 continue; 1408 rxq = &rxq_ctrl->rxq; 1409 n_ibv++; 1410 desc += 1 << rxq->elts_n; 1411 /* Get the max number of strides. */ 1412 if (log_strd_num < rxq->log_strd_num) 1413 log_strd_num = rxq->log_strd_num; 1414 /* Get the max size of a stride. */ 1415 if (log_strd_sz < rxq->log_strd_sz) 1416 log_strd_sz = rxq->log_strd_sz; 1417 } 1418 MLX5_ASSERT(log_strd_num && log_strd_sz); 1419 buf_len = RTE_BIT32(log_strd_num) * RTE_BIT32(log_strd_sz); 1420 obj_size = sizeof(struct mlx5_mprq_buf) + buf_len + 1421 RTE_BIT32(log_strd_num) * 1422 sizeof(struct rte_mbuf_ext_shared_info) + 1423 RTE_PKTMBUF_HEADROOM; 1424 /* 1425 * Received packets can be either memcpy'd or externally referenced. In 1426 * case that the packet is attached to an mbuf as an external buffer, as 1427 * it isn't possible to predict how the buffers will be queued by 1428 * application, there's no option to exactly pre-allocate needed buffers 1429 * in advance but to speculatively prepares enough buffers. 1430 * 1431 * In the data path, if this Mempool is depleted, PMD will try to memcpy 1432 * received packets to buffers provided by application (rxq->mp) until 1433 * this Mempool gets available again. 1434 */ 1435 desc *= 4; 1436 obj_num = desc + MLX5_MPRQ_MP_CACHE_SZ * n_ibv; 1437 /* 1438 * rte_mempool_create_empty() has sanity check to refuse large cache 1439 * size compared to the number of elements. 1440 * CACHE_FLUSHTHRESH_MULTIPLIER is defined in a C file, so using a 1441 * constant number 2 instead. 1442 */ 1443 obj_num = RTE_MAX(obj_num, MLX5_MPRQ_MP_CACHE_SZ * 2); 1444 /* Check a mempool is already allocated and if it can be resued. */ 1445 if (mp != NULL && mp->elt_size >= obj_size && mp->size >= obj_num) { 1446 DRV_LOG(DEBUG, "port %u mempool %s is being reused", 1447 dev->data->port_id, mp->name); 1448 /* Reuse. */ 1449 goto exit; 1450 } else if (mp != NULL) { 1451 DRV_LOG(DEBUG, "port %u mempool %s should be resized, freeing it", 1452 dev->data->port_id, mp->name); 1453 /* 1454 * If failed to free, which means it may be still in use, no way 1455 * but to keep using the existing one. On buffer underrun, 1456 * packets will be memcpy'd instead of external buffer 1457 * attachment. 1458 */ 1459 if (mlx5_mprq_free_mp(dev)) { 1460 if (mp->elt_size >= obj_size) 1461 goto exit; 1462 else 1463 return -rte_errno; 1464 } 1465 } 1466 snprintf(name, sizeof(name), "port-%u-mprq", dev->data->port_id); 1467 mp = rte_mempool_create(name, obj_num, obj_size, MLX5_MPRQ_MP_CACHE_SZ, 1468 0, NULL, NULL, mlx5_mprq_buf_init, 1469 (void *)((uintptr_t)1 << log_strd_num), 1470 dev->device->numa_node, 0); 1471 if (mp == NULL) { 1472 DRV_LOG(ERR, 1473 "port %u failed to allocate a mempool for" 1474 " Multi-Packet RQ, count=%u, size=%u", 1475 dev->data->port_id, obj_num, obj_size); 1476 rte_errno = ENOMEM; 1477 return -rte_errno; 1478 } 1479 ret = mlx5_mr_mempool_register(priv->sh->cdev, mp, false); 1480 if (ret < 0 && rte_errno != EEXIST) { 1481 ret = rte_errno; 1482 DRV_LOG(ERR, "port %u failed to register a mempool for Multi-Packet RQ", 1483 dev->data->port_id); 1484 rte_mempool_free(mp); 1485 rte_errno = ret; 1486 return -rte_errno; 1487 } 1488 priv->mprq_mp = mp; 1489 exit: 1490 /* Set mempool for each Rx queue. */ 1491 for (i = 0; i != priv->rxqs_n; ++i) { 1492 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); 1493 1494 if (rxq_ctrl == NULL || rxq_ctrl->is_hairpin) 1495 continue; 1496 rxq_ctrl->rxq.mprq_mp = mp; 1497 } 1498 DRV_LOG(INFO, "port %u Multi-Packet RQ is configured", 1499 dev->data->port_id); 1500 return 0; 1501 } 1502 1503 #define MLX5_MAX_TCP_HDR_OFFSET ((unsigned int)(sizeof(struct rte_ether_hdr) + \ 1504 sizeof(struct rte_vlan_hdr) * 2 + \ 1505 sizeof(struct rte_ipv6_hdr))) 1506 #define MAX_TCP_OPTION_SIZE 40u 1507 #define MLX5_MAX_LRO_HEADER_FIX ((unsigned int)(MLX5_MAX_TCP_HDR_OFFSET + \ 1508 sizeof(struct rte_tcp_hdr) + \ 1509 MAX_TCP_OPTION_SIZE)) 1510 1511 /** 1512 * Adjust the maximum LRO massage size. 1513 * 1514 * @param dev 1515 * Pointer to Ethernet device. 1516 * @param idx 1517 * RX queue index. 1518 * @param max_lro_size 1519 * The maximum size for LRO packet. 1520 */ 1521 static void 1522 mlx5_max_lro_msg_size_adjust(struct rte_eth_dev *dev, uint16_t idx, 1523 uint32_t max_lro_size) 1524 { 1525 struct mlx5_priv *priv = dev->data->dev_private; 1526 1527 if (priv->sh->cdev->config.hca_attr.lro_max_msg_sz_mode == 1528 MLX5_LRO_MAX_MSG_SIZE_START_FROM_L4 && max_lro_size > 1529 MLX5_MAX_TCP_HDR_OFFSET) 1530 max_lro_size -= MLX5_MAX_TCP_HDR_OFFSET; 1531 max_lro_size = RTE_MIN(max_lro_size, MLX5_MAX_LRO_SIZE); 1532 MLX5_ASSERT(max_lro_size >= MLX5_LRO_SEG_CHUNK_SIZE); 1533 max_lro_size /= MLX5_LRO_SEG_CHUNK_SIZE; 1534 if (priv->max_lro_msg_size) 1535 priv->max_lro_msg_size = 1536 RTE_MIN((uint32_t)priv->max_lro_msg_size, max_lro_size); 1537 else 1538 priv->max_lro_msg_size = max_lro_size; 1539 DRV_LOG(DEBUG, 1540 "port %u Rx Queue %u max LRO message size adjusted to %u bytes", 1541 dev->data->port_id, idx, 1542 priv->max_lro_msg_size * MLX5_LRO_SEG_CHUNK_SIZE); 1543 } 1544 1545 /** 1546 * Prepare both size and number of stride for Multi-Packet RQ. 1547 * 1548 * @param dev 1549 * Pointer to Ethernet device. 1550 * @param idx 1551 * RX queue index. 1552 * @param desc 1553 * Number of descriptors to configure in queue. 1554 * @param rx_seg_en 1555 * Indicator if Rx segment enables, if so Multi-Packet RQ doesn't enable. 1556 * @param min_mbuf_size 1557 * Non scatter min mbuf size, max_rx_pktlen plus overhead. 1558 * @param actual_log_stride_num 1559 * Log number of strides to configure for this queue. 1560 * @param actual_log_stride_size 1561 * Log stride size to configure for this queue. 1562 * @param is_extmem 1563 * Is external pinned memory pool used. 1564 * @return 1565 * 0 if Multi-Packet RQ is supported, otherwise -1. 1566 */ 1567 static int 1568 mlx5_mprq_prepare(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 1569 bool rx_seg_en, uint32_t min_mbuf_size, 1570 uint32_t *actual_log_stride_num, 1571 uint32_t *actual_log_stride_size, 1572 bool is_extmem) 1573 { 1574 struct mlx5_priv *priv = dev->data->dev_private; 1575 struct mlx5_port_config *config = &priv->config; 1576 struct mlx5_dev_cap *dev_cap = &priv->sh->dev_cap; 1577 uint32_t log_min_stride_num = dev_cap->mprq.log_min_stride_num; 1578 uint32_t log_max_stride_num = dev_cap->mprq.log_max_stride_num; 1579 uint32_t log_def_stride_num = 1580 RTE_MIN(RTE_MAX(MLX5_MPRQ_DEFAULT_LOG_STRIDE_NUM, 1581 log_min_stride_num), 1582 log_max_stride_num); 1583 uint32_t log_min_stride_size = dev_cap->mprq.log_min_stride_size; 1584 uint32_t log_max_stride_size = dev_cap->mprq.log_max_stride_size; 1585 uint32_t log_def_stride_size = 1586 RTE_MIN(RTE_MAX(MLX5_MPRQ_DEFAULT_LOG_STRIDE_SIZE, 1587 log_min_stride_size), 1588 log_max_stride_size); 1589 uint32_t log_stride_wqe_size; 1590 1591 if (mlx5_check_mprq_support(dev) != 1 || rx_seg_en || is_extmem) 1592 goto unsupport; 1593 /* Checks if chosen number of strides is in supported range. */ 1594 if (config->mprq.log_stride_num > log_max_stride_num || 1595 config->mprq.log_stride_num < log_min_stride_num) { 1596 *actual_log_stride_num = log_def_stride_num; 1597 DRV_LOG(WARNING, 1598 "Port %u Rx queue %u number of strides for Multi-Packet RQ is out of range, setting default value (%u)", 1599 dev->data->port_id, idx, RTE_BIT32(log_def_stride_num)); 1600 } else { 1601 *actual_log_stride_num = config->mprq.log_stride_num; 1602 } 1603 if (config->mprq.log_stride_size) { 1604 /* Checks if chosen size of stride is in supported range. */ 1605 if (config->mprq.log_stride_size > log_max_stride_size || 1606 config->mprq.log_stride_size < log_min_stride_size) { 1607 *actual_log_stride_size = log_def_stride_size; 1608 DRV_LOG(WARNING, 1609 "Port %u Rx queue %u size of a stride for Multi-Packet RQ is out of range, setting default value (%u)", 1610 dev->data->port_id, idx, 1611 RTE_BIT32(log_def_stride_size)); 1612 } else { 1613 *actual_log_stride_size = config->mprq.log_stride_size; 1614 } 1615 } else { 1616 if (min_mbuf_size <= RTE_BIT32(log_max_stride_size)) 1617 *actual_log_stride_size = log2above(min_mbuf_size); 1618 else 1619 goto unsupport; 1620 } 1621 log_stride_wqe_size = *actual_log_stride_num + *actual_log_stride_size; 1622 /* Check if WQE buffer size is supported by hardware. */ 1623 if (log_stride_wqe_size < dev_cap->mprq.log_min_stride_wqe_size) { 1624 *actual_log_stride_num = log_def_stride_num; 1625 *actual_log_stride_size = log_def_stride_size; 1626 DRV_LOG(WARNING, 1627 "Port %u Rx queue %u size of WQE buffer for Multi-Packet RQ is too small, setting default values (stride_num_n=%u, stride_size_n=%u)", 1628 dev->data->port_id, idx, RTE_BIT32(log_def_stride_num), 1629 RTE_BIT32(log_def_stride_size)); 1630 log_stride_wqe_size = log_def_stride_num + log_def_stride_size; 1631 } 1632 MLX5_ASSERT(log_stride_wqe_size >= 1633 dev_cap->mprq.log_min_stride_wqe_size); 1634 if (desc <= RTE_BIT32(*actual_log_stride_num)) 1635 goto unsupport; 1636 if (min_mbuf_size > RTE_BIT32(log_stride_wqe_size)) { 1637 DRV_LOG(WARNING, "Port %u Rx queue %u " 1638 "Multi-Packet RQ is unsupported, WQE buffer size (%u) " 1639 "is smaller than min mbuf size (%u)", 1640 dev->data->port_id, idx, RTE_BIT32(log_stride_wqe_size), 1641 min_mbuf_size); 1642 goto unsupport; 1643 } 1644 DRV_LOG(DEBUG, "Port %u Rx queue %u " 1645 "Multi-Packet RQ is enabled strd_num_n = %u, strd_sz_n = %u", 1646 dev->data->port_id, idx, RTE_BIT32(*actual_log_stride_num), 1647 RTE_BIT32(*actual_log_stride_size)); 1648 return 0; 1649 unsupport: 1650 if (config->mprq.enabled) 1651 DRV_LOG(WARNING, 1652 "Port %u MPRQ is requested but cannot be enabled\n" 1653 " (requested: pkt_sz = %u, desc_num = %u," 1654 " rxq_num = %u, stride_sz = %u, stride_num = %u\n" 1655 " supported: min_rxqs_num = %u, min_buf_wqe_sz = %u" 1656 " min_stride_sz = %u, max_stride_sz = %u).\n" 1657 "Rx segment is %senabled. External mempool is %sused.", 1658 dev->data->port_id, min_mbuf_size, desc, priv->rxqs_n, 1659 RTE_BIT32(config->mprq.log_stride_size), 1660 RTE_BIT32(config->mprq.log_stride_num), 1661 config->mprq.min_rxqs_num, 1662 RTE_BIT32(dev_cap->mprq.log_min_stride_wqe_size), 1663 RTE_BIT32(dev_cap->mprq.log_min_stride_size), 1664 RTE_BIT32(dev_cap->mprq.log_max_stride_size), 1665 rx_seg_en ? "" : "not ", is_extmem ? "" : "not "); 1666 return -1; 1667 } 1668 1669 /** 1670 * Create a DPDK Rx queue. 1671 * 1672 * @param dev 1673 * Pointer to Ethernet device. 1674 * @param idx 1675 * RX queue index. 1676 * @param desc 1677 * Number of descriptors to configure in queue. 1678 * @param socket 1679 * NUMA socket on which memory must be allocated. 1680 * 1681 * @return 1682 * A DPDK queue object on success, NULL otherwise and rte_errno is set. 1683 */ 1684 struct mlx5_rxq_ctrl * 1685 mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 1686 unsigned int socket, const struct rte_eth_rxconf *conf, 1687 const struct rte_eth_rxseg_split *rx_seg, uint16_t n_seg, 1688 bool is_extmem) 1689 { 1690 struct mlx5_priv *priv = dev->data->dev_private; 1691 struct mlx5_rxq_ctrl *tmpl; 1692 unsigned int mb_len = rte_pktmbuf_data_room_size(rx_seg[0].mp); 1693 struct mlx5_port_config *config = &priv->config; 1694 uint64_t offloads = conf->offloads | 1695 dev->data->dev_conf.rxmode.offloads; 1696 unsigned int lro_on_queue = !!(offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO); 1697 unsigned int max_rx_pktlen = lro_on_queue ? 1698 dev->data->dev_conf.rxmode.max_lro_pkt_size : 1699 dev->data->mtu + (unsigned int)RTE_ETHER_HDR_LEN + 1700 RTE_ETHER_CRC_LEN; 1701 unsigned int non_scatter_min_mbuf_size = max_rx_pktlen + 1702 RTE_PKTMBUF_HEADROOM; 1703 unsigned int max_lro_size = 0; 1704 unsigned int first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM; 1705 uint32_t mprq_log_actual_stride_num = 0; 1706 uint32_t mprq_log_actual_stride_size = 0; 1707 bool rx_seg_en = n_seg != 1 || rx_seg[0].offset || rx_seg[0].length; 1708 const int mprq_en = !mlx5_mprq_prepare(dev, idx, desc, rx_seg_en, 1709 non_scatter_min_mbuf_size, 1710 &mprq_log_actual_stride_num, 1711 &mprq_log_actual_stride_size, 1712 is_extmem); 1713 /* 1714 * Always allocate extra slots, even if eventually 1715 * the vector Rx will not be used. 1716 */ 1717 uint16_t desc_n = desc + config->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP; 1718 size_t alloc_size = sizeof(*tmpl) + desc_n * sizeof(struct rte_mbuf *); 1719 const struct rte_eth_rxseg_split *qs_seg = rx_seg; 1720 unsigned int tail_len; 1721 1722 if (mprq_en) { 1723 /* Trim the number of descs needed. */ 1724 desc >>= mprq_log_actual_stride_num; 1725 alloc_size += desc * sizeof(struct mlx5_mprq_buf *); 1726 } 1727 tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, alloc_size, 0, socket); 1728 if (!tmpl) { 1729 rte_errno = ENOMEM; 1730 return NULL; 1731 } 1732 LIST_INIT(&tmpl->owners); 1733 MLX5_ASSERT(n_seg && n_seg <= MLX5_MAX_RXQ_NSEG); 1734 /* 1735 * Save the original segment configuration in the shared queue 1736 * descriptor for the later check on the sibling queue creation. 1737 */ 1738 tmpl->rxseg_n = n_seg; 1739 rte_memcpy(tmpl->rxseg, qs_seg, 1740 sizeof(struct rte_eth_rxseg_split) * n_seg); 1741 /* 1742 * Build the array of actual buffer offsets and lengths. 1743 * Pad with the buffers from the last memory pool if 1744 * needed to handle max size packets, replace zero length 1745 * with the buffer length from the pool. 1746 */ 1747 tail_len = max_rx_pktlen; 1748 do { 1749 struct mlx5_eth_rxseg *hw_seg = 1750 &tmpl->rxq.rxseg[tmpl->rxq.rxseg_n]; 1751 uint32_t buf_len, offset, seg_len; 1752 1753 /* 1754 * For the buffers beyond descriptions offset is zero, 1755 * the first buffer contains head room. 1756 */ 1757 buf_len = rte_pktmbuf_data_room_size(qs_seg->mp); 1758 offset = (tmpl->rxq.rxseg_n >= n_seg ? 0 : qs_seg->offset) + 1759 (tmpl->rxq.rxseg_n ? 0 : RTE_PKTMBUF_HEADROOM); 1760 /* 1761 * For the buffers beyond descriptions the length is 1762 * pool buffer length, zero lengths are replaced with 1763 * pool buffer length either. 1764 */ 1765 seg_len = tmpl->rxq.rxseg_n >= n_seg ? buf_len : 1766 qs_seg->length ? 1767 qs_seg->length : 1768 (buf_len - offset); 1769 /* Check is done in long int, now overflows. */ 1770 if (buf_len < seg_len + offset) { 1771 DRV_LOG(ERR, "port %u Rx queue %u: Split offset/length " 1772 "%u/%u can't be satisfied", 1773 dev->data->port_id, idx, 1774 qs_seg->length, qs_seg->offset); 1775 rte_errno = EINVAL; 1776 goto error; 1777 } 1778 if (seg_len > tail_len) 1779 seg_len = buf_len - offset; 1780 if (++tmpl->rxq.rxseg_n > MLX5_MAX_RXQ_NSEG) { 1781 DRV_LOG(ERR, 1782 "port %u too many SGEs (%u) needed to handle" 1783 " requested maximum packet size %u, the maximum" 1784 " supported are %u", dev->data->port_id, 1785 tmpl->rxq.rxseg_n, max_rx_pktlen, 1786 MLX5_MAX_RXQ_NSEG); 1787 rte_errno = ENOTSUP; 1788 goto error; 1789 } 1790 /* Build the actual scattering element in the queue object. */ 1791 hw_seg->mp = qs_seg->mp; 1792 MLX5_ASSERT(offset <= UINT16_MAX); 1793 MLX5_ASSERT(seg_len <= UINT16_MAX); 1794 hw_seg->offset = (uint16_t)offset; 1795 hw_seg->length = (uint16_t)seg_len; 1796 /* 1797 * Advance the segment descriptor, the padding is the based 1798 * on the attributes of the last descriptor. 1799 */ 1800 if (tmpl->rxq.rxseg_n < n_seg) 1801 qs_seg++; 1802 tail_len -= RTE_MIN(tail_len, seg_len); 1803 } while (tail_len || !rte_is_power_of_2(tmpl->rxq.rxseg_n)); 1804 MLX5_ASSERT(tmpl->rxq.rxseg_n && 1805 tmpl->rxq.rxseg_n <= MLX5_MAX_RXQ_NSEG); 1806 if (tmpl->rxq.rxseg_n > 1 && !(offloads & RTE_ETH_RX_OFFLOAD_SCATTER)) { 1807 DRV_LOG(ERR, "port %u Rx queue %u: Scatter offload is not" 1808 " configured and no enough mbuf space(%u) to contain " 1809 "the maximum RX packet length(%u) with head-room(%u)", 1810 dev->data->port_id, idx, mb_len, max_rx_pktlen, 1811 RTE_PKTMBUF_HEADROOM); 1812 rte_errno = ENOSPC; 1813 goto error; 1814 } 1815 tmpl->is_hairpin = false; 1816 if (mlx5_mr_ctrl_init(&tmpl->rxq.mr_ctrl, 1817 &priv->sh->cdev->mr_scache.dev_gen, socket)) { 1818 /* rte_errno is already set. */ 1819 goto error; 1820 } 1821 tmpl->socket = socket; 1822 if (dev->data->dev_conf.intr_conf.rxq) 1823 tmpl->irq = 1; 1824 if (mprq_en) { 1825 /* TODO: Rx scatter isn't supported yet. */ 1826 tmpl->rxq.sges_n = 0; 1827 tmpl->rxq.log_strd_num = mprq_log_actual_stride_num; 1828 tmpl->rxq.log_strd_sz = mprq_log_actual_stride_size; 1829 tmpl->rxq.strd_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT; 1830 tmpl->rxq.strd_scatter_en = 1831 !!(offloads & RTE_ETH_RX_OFFLOAD_SCATTER); 1832 tmpl->rxq.mprq_max_memcpy_len = RTE_MIN(first_mb_free_size, 1833 config->mprq.max_memcpy_len); 1834 max_lro_size = RTE_MIN(max_rx_pktlen, 1835 RTE_BIT32(tmpl->rxq.log_strd_num) * 1836 RTE_BIT32(tmpl->rxq.log_strd_sz)); 1837 } else if (tmpl->rxq.rxseg_n == 1) { 1838 MLX5_ASSERT(max_rx_pktlen <= first_mb_free_size); 1839 tmpl->rxq.sges_n = 0; 1840 max_lro_size = max_rx_pktlen; 1841 } else if (offloads & RTE_ETH_RX_OFFLOAD_SCATTER) { 1842 unsigned int sges_n; 1843 1844 if (lro_on_queue && first_mb_free_size < 1845 MLX5_MAX_LRO_HEADER_FIX) { 1846 DRV_LOG(ERR, "Not enough space in the first segment(%u)" 1847 " to include the max header size(%u) for LRO", 1848 first_mb_free_size, MLX5_MAX_LRO_HEADER_FIX); 1849 rte_errno = ENOTSUP; 1850 goto error; 1851 } 1852 /* 1853 * Determine the number of SGEs needed for a full packet 1854 * and round it to the next power of two. 1855 */ 1856 sges_n = log2above(tmpl->rxq.rxseg_n); 1857 if (sges_n > MLX5_MAX_LOG_RQ_SEGS) { 1858 DRV_LOG(ERR, 1859 "port %u too many SGEs (%u) needed to handle" 1860 " requested maximum packet size %u, the maximum" 1861 " supported are %u", dev->data->port_id, 1862 1 << sges_n, max_rx_pktlen, 1863 1u << MLX5_MAX_LOG_RQ_SEGS); 1864 rte_errno = ENOTSUP; 1865 goto error; 1866 } 1867 tmpl->rxq.sges_n = sges_n; 1868 max_lro_size = max_rx_pktlen; 1869 } 1870 DRV_LOG(DEBUG, "port %u maximum number of segments per packet: %u", 1871 dev->data->port_id, 1 << tmpl->rxq.sges_n); 1872 if (desc % (1 << tmpl->rxq.sges_n)) { 1873 DRV_LOG(ERR, 1874 "port %u number of Rx queue descriptors (%u) is not a" 1875 " multiple of SGEs per packet (%u)", 1876 dev->data->port_id, 1877 desc, 1878 1 << tmpl->rxq.sges_n); 1879 rte_errno = EINVAL; 1880 goto error; 1881 } 1882 mlx5_max_lro_msg_size_adjust(dev, idx, max_lro_size); 1883 /* Toggle RX checksum offload if hardware supports it. */ 1884 tmpl->rxq.csum = !!(offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM); 1885 /* Configure Rx timestamp. */ 1886 tmpl->rxq.hw_timestamp = !!(offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP); 1887 tmpl->rxq.timestamp_rx_flag = 0; 1888 if (tmpl->rxq.hw_timestamp && rte_mbuf_dyn_rx_timestamp_register( 1889 &tmpl->rxq.timestamp_offset, 1890 &tmpl->rxq.timestamp_rx_flag) != 0) { 1891 DRV_LOG(ERR, "Cannot register Rx timestamp field/flag"); 1892 goto error; 1893 } 1894 /* Configure VLAN stripping. */ 1895 tmpl->rxq.vlan_strip = !!(offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP); 1896 /* By default, FCS (CRC) is stripped by hardware. */ 1897 tmpl->rxq.crc_present = 0; 1898 tmpl->rxq.lro = lro_on_queue; 1899 if (offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) { 1900 if (priv->sh->config.hw_fcs_strip) { 1901 /* 1902 * RQs used for LRO-enabled TIRs should not be 1903 * configured to scatter the FCS. 1904 */ 1905 if (lro_on_queue) 1906 DRV_LOG(WARNING, 1907 "port %u CRC stripping has been " 1908 "disabled but will still be performed " 1909 "by hardware, because LRO is enabled", 1910 dev->data->port_id); 1911 else 1912 tmpl->rxq.crc_present = 1; 1913 } else { 1914 DRV_LOG(WARNING, 1915 "port %u CRC stripping has been disabled but will" 1916 " still be performed by hardware, make sure MLNX_OFED" 1917 " and firmware are up to date", 1918 dev->data->port_id); 1919 } 1920 } 1921 DRV_LOG(DEBUG, 1922 "port %u CRC stripping is %s, %u bytes will be subtracted from" 1923 " incoming frames to hide it", 1924 dev->data->port_id, 1925 tmpl->rxq.crc_present ? "disabled" : "enabled", 1926 tmpl->rxq.crc_present << 2); 1927 tmpl->rxq.rss_hash = !!priv->rss_conf.rss_hf && 1928 (!!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS)); 1929 /* Save port ID. */ 1930 tmpl->rxq.port_id = dev->data->port_id; 1931 tmpl->sh = priv->sh; 1932 tmpl->rxq.mp = rx_seg[0].mp; 1933 tmpl->rxq.elts_n = log2above(desc); 1934 tmpl->rxq.rq_repl_thresh = MLX5_VPMD_RXQ_RPLNSH_THRESH(desc_n); 1935 tmpl->rxq.elts = (struct rte_mbuf *(*)[desc_n])(tmpl + 1); 1936 tmpl->rxq.mprq_bufs = 1937 (struct mlx5_mprq_buf *(*)[desc])(*tmpl->rxq.elts + desc_n); 1938 tmpl->rxq.idx = idx; 1939 if (conf->share_group > 0) { 1940 tmpl->rxq.shared = 1; 1941 tmpl->share_group = conf->share_group; 1942 tmpl->share_qid = conf->share_qid; 1943 LIST_INSERT_HEAD(&priv->sh->shared_rxqs, tmpl, share_entry); 1944 } 1945 LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next); 1946 return tmpl; 1947 error: 1948 mlx5_mr_btree_free(&tmpl->rxq.mr_ctrl.cache_bh); 1949 mlx5_free(tmpl); 1950 return NULL; 1951 } 1952 1953 /** 1954 * Create a DPDK Rx hairpin queue. 1955 * 1956 * @param dev 1957 * Pointer to Ethernet device. 1958 * @param rxq 1959 * RX queue. 1960 * @param desc 1961 * Number of descriptors to configure in queue. 1962 * @param hairpin_conf 1963 * The hairpin binding configuration. 1964 * 1965 * @return 1966 * A DPDK queue object on success, NULL otherwise and rte_errno is set. 1967 */ 1968 struct mlx5_rxq_ctrl * 1969 mlx5_rxq_hairpin_new(struct rte_eth_dev *dev, struct mlx5_rxq_priv *rxq, 1970 uint16_t desc, 1971 const struct rte_eth_hairpin_conf *hairpin_conf) 1972 { 1973 uint16_t idx = rxq->idx; 1974 struct mlx5_priv *priv = dev->data->dev_private; 1975 struct mlx5_rxq_ctrl *tmpl; 1976 1977 tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, sizeof(*tmpl), 0, 1978 SOCKET_ID_ANY); 1979 if (!tmpl) { 1980 rte_errno = ENOMEM; 1981 return NULL; 1982 } 1983 LIST_INIT(&tmpl->owners); 1984 rxq->ctrl = tmpl; 1985 LIST_INSERT_HEAD(&tmpl->owners, rxq, owner_entry); 1986 tmpl->is_hairpin = true; 1987 tmpl->socket = SOCKET_ID_ANY; 1988 tmpl->rxq.rss_hash = 0; 1989 tmpl->rxq.port_id = dev->data->port_id; 1990 tmpl->sh = priv->sh; 1991 tmpl->rxq.mp = NULL; 1992 tmpl->rxq.elts_n = log2above(desc); 1993 tmpl->rxq.elts = NULL; 1994 tmpl->rxq.mr_ctrl.cache_bh = (struct mlx5_mr_btree) { 0 }; 1995 tmpl->rxq.idx = idx; 1996 rxq->hairpin_conf = *hairpin_conf; 1997 mlx5_rxq_ref(dev, idx); 1998 LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next); 1999 return tmpl; 2000 } 2001 2002 /** 2003 * Increase Rx queue reference count. 2004 * 2005 * @param dev 2006 * Pointer to Ethernet device. 2007 * @param idx 2008 * RX queue index. 2009 * 2010 * @return 2011 * A pointer to the queue if it exists, NULL otherwise. 2012 */ 2013 struct mlx5_rxq_priv * 2014 mlx5_rxq_ref(struct rte_eth_dev *dev, uint16_t idx) 2015 { 2016 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 2017 2018 if (rxq != NULL) 2019 __atomic_fetch_add(&rxq->refcnt, 1, __ATOMIC_RELAXED); 2020 return rxq; 2021 } 2022 2023 /** 2024 * Dereference a Rx queue. 2025 * 2026 * @param dev 2027 * Pointer to Ethernet device. 2028 * @param idx 2029 * RX queue index. 2030 * 2031 * @return 2032 * Updated reference count. 2033 */ 2034 uint32_t 2035 mlx5_rxq_deref(struct rte_eth_dev *dev, uint16_t idx) 2036 { 2037 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 2038 2039 if (rxq == NULL) 2040 return 0; 2041 return __atomic_sub_fetch(&rxq->refcnt, 1, __ATOMIC_RELAXED); 2042 } 2043 2044 /** 2045 * Get a Rx queue. 2046 * 2047 * @param dev 2048 * Pointer to Ethernet device. 2049 * @param idx 2050 * RX queue index. 2051 * 2052 * @return 2053 * A pointer to the queue if it exists, NULL otherwise. 2054 */ 2055 struct mlx5_rxq_priv * 2056 mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx) 2057 { 2058 struct mlx5_priv *priv = dev->data->dev_private; 2059 2060 if (idx >= priv->rxqs_n) 2061 return NULL; 2062 MLX5_ASSERT(priv->rxq_privs != NULL); 2063 return (*priv->rxq_privs)[idx]; 2064 } 2065 2066 /** 2067 * Get Rx queue shareable control. 2068 * 2069 * @param dev 2070 * Pointer to Ethernet device. 2071 * @param idx 2072 * RX queue index. 2073 * 2074 * @return 2075 * A pointer to the queue control if it exists, NULL otherwise. 2076 */ 2077 struct mlx5_rxq_ctrl * 2078 mlx5_rxq_ctrl_get(struct rte_eth_dev *dev, uint16_t idx) 2079 { 2080 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 2081 2082 return rxq == NULL ? NULL : rxq->ctrl; 2083 } 2084 2085 /** 2086 * Get Rx queue shareable data. 2087 * 2088 * @param dev 2089 * Pointer to Ethernet device. 2090 * @param idx 2091 * RX queue index. 2092 * 2093 * @return 2094 * A pointer to the queue data if it exists, NULL otherwise. 2095 */ 2096 struct mlx5_rxq_data * 2097 mlx5_rxq_data_get(struct rte_eth_dev *dev, uint16_t idx) 2098 { 2099 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 2100 2101 return rxq == NULL ? NULL : &rxq->ctrl->rxq; 2102 } 2103 2104 /** 2105 * Increase an external Rx queue reference count. 2106 * 2107 * @param dev 2108 * Pointer to Ethernet device. 2109 * @param idx 2110 * External RX queue index. 2111 * 2112 * @return 2113 * A pointer to the queue if it exists, NULL otherwise. 2114 */ 2115 struct mlx5_external_rxq * 2116 mlx5_ext_rxq_ref(struct rte_eth_dev *dev, uint16_t idx) 2117 { 2118 struct mlx5_external_rxq *rxq = mlx5_ext_rxq_get(dev, idx); 2119 2120 __atomic_fetch_add(&rxq->refcnt, 1, __ATOMIC_RELAXED); 2121 return rxq; 2122 } 2123 2124 /** 2125 * Decrease an external Rx queue reference count. 2126 * 2127 * @param dev 2128 * Pointer to Ethernet device. 2129 * @param idx 2130 * External RX queue index. 2131 * 2132 * @return 2133 * Updated reference count. 2134 */ 2135 uint32_t 2136 mlx5_ext_rxq_deref(struct rte_eth_dev *dev, uint16_t idx) 2137 { 2138 struct mlx5_external_rxq *rxq = mlx5_ext_rxq_get(dev, idx); 2139 2140 return __atomic_sub_fetch(&rxq->refcnt, 1, __ATOMIC_RELAXED); 2141 } 2142 2143 /** 2144 * Get an external Rx queue. 2145 * 2146 * @param dev 2147 * Pointer to Ethernet device. 2148 * @param idx 2149 * External Rx queue index. 2150 * 2151 * @return 2152 * A pointer to the queue if it exists, NULL otherwise. 2153 */ 2154 struct mlx5_external_rxq * 2155 mlx5_ext_rxq_get(struct rte_eth_dev *dev, uint16_t idx) 2156 { 2157 struct mlx5_priv *priv = dev->data->dev_private; 2158 2159 MLX5_ASSERT(mlx5_is_external_rxq(dev, idx)); 2160 return &priv->ext_rxqs[idx - MLX5_EXTERNAL_RX_QUEUE_ID_MIN]; 2161 } 2162 2163 /** 2164 * Dereference a list of Rx queues. 2165 * 2166 * @param dev 2167 * Pointer to Ethernet device. 2168 * @param queues 2169 * List of Rx queues to deref. 2170 * @param queues_n 2171 * Number of queues in the array. 2172 */ 2173 static void 2174 mlx5_rxqs_deref(struct rte_eth_dev *dev, uint16_t *queues, 2175 const uint32_t queues_n) 2176 { 2177 uint32_t i; 2178 2179 for (i = 0; i < queues_n; i++) { 2180 if (mlx5_is_external_rxq(dev, queues[i])) 2181 claim_nonzero(mlx5_ext_rxq_deref(dev, queues[i])); 2182 else 2183 claim_nonzero(mlx5_rxq_deref(dev, queues[i])); 2184 } 2185 } 2186 2187 /** 2188 * Increase reference count for list of Rx queues. 2189 * 2190 * @param dev 2191 * Pointer to Ethernet device. 2192 * @param queues 2193 * List of Rx queues to ref. 2194 * @param queues_n 2195 * Number of queues in the array. 2196 * 2197 * @return 2198 * 0 on success, a negative errno value otherwise and rte_errno is set. 2199 */ 2200 static int 2201 mlx5_rxqs_ref(struct rte_eth_dev *dev, uint16_t *queues, 2202 const uint32_t queues_n) 2203 { 2204 uint32_t i; 2205 2206 for (i = 0; i != queues_n; ++i) { 2207 if (mlx5_is_external_rxq(dev, queues[i])) { 2208 if (mlx5_ext_rxq_ref(dev, queues[i]) == NULL) 2209 goto error; 2210 } else { 2211 if (mlx5_rxq_ref(dev, queues[i]) == NULL) 2212 goto error; 2213 } 2214 } 2215 return 0; 2216 error: 2217 mlx5_rxqs_deref(dev, queues, i); 2218 rte_errno = EINVAL; 2219 return -rte_errno; 2220 } 2221 2222 /** 2223 * Release a Rx queue. 2224 * 2225 * @param dev 2226 * Pointer to Ethernet device. 2227 * @param idx 2228 * RX queue index. 2229 * 2230 * @return 2231 * 1 while a reference on it exists, 0 when freed. 2232 */ 2233 int 2234 mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) 2235 { 2236 struct mlx5_priv *priv = dev->data->dev_private; 2237 struct mlx5_rxq_priv *rxq; 2238 struct mlx5_rxq_ctrl *rxq_ctrl; 2239 uint32_t refcnt; 2240 2241 if (priv->rxq_privs == NULL) 2242 return 0; 2243 rxq = mlx5_rxq_get(dev, idx); 2244 if (rxq == NULL || rxq->refcnt == 0) 2245 return 0; 2246 rxq_ctrl = rxq->ctrl; 2247 refcnt = mlx5_rxq_deref(dev, idx); 2248 if (refcnt > 1) { 2249 return 1; 2250 } else if (refcnt == 1) { /* RxQ stopped. */ 2251 priv->obj_ops.rxq_obj_release(rxq); 2252 if (!rxq_ctrl->started && rxq_ctrl->obj != NULL) { 2253 LIST_REMOVE(rxq_ctrl->obj, next); 2254 mlx5_free(rxq_ctrl->obj); 2255 rxq_ctrl->obj = NULL; 2256 } 2257 if (!rxq_ctrl->is_hairpin) { 2258 if (!rxq_ctrl->started) 2259 rxq_free_elts(rxq_ctrl); 2260 dev->data->rx_queue_state[idx] = 2261 RTE_ETH_QUEUE_STATE_STOPPED; 2262 } 2263 } else { /* Refcnt zero, closing device. */ 2264 LIST_REMOVE(rxq, owner_entry); 2265 if (LIST_EMPTY(&rxq_ctrl->owners)) { 2266 if (!rxq_ctrl->is_hairpin) 2267 mlx5_mr_btree_free 2268 (&rxq_ctrl->rxq.mr_ctrl.cache_bh); 2269 if (rxq_ctrl->rxq.shared) 2270 LIST_REMOVE(rxq_ctrl, share_entry); 2271 LIST_REMOVE(rxq_ctrl, next); 2272 mlx5_free(rxq_ctrl); 2273 } 2274 dev->data->rx_queues[idx] = NULL; 2275 mlx5_free(rxq); 2276 (*priv->rxq_privs)[idx] = NULL; 2277 } 2278 return 0; 2279 } 2280 2281 /** 2282 * Verify the Rx Queue list is empty 2283 * 2284 * @param dev 2285 * Pointer to Ethernet device. 2286 * 2287 * @return 2288 * The number of object not released. 2289 */ 2290 int 2291 mlx5_rxq_verify(struct rte_eth_dev *dev) 2292 { 2293 struct mlx5_priv *priv = dev->data->dev_private; 2294 struct mlx5_rxq_ctrl *rxq_ctrl; 2295 int ret = 0; 2296 2297 LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { 2298 DRV_LOG(DEBUG, "port %u Rx Queue %u still referenced", 2299 dev->data->port_id, rxq_ctrl->rxq.idx); 2300 ++ret; 2301 } 2302 return ret; 2303 } 2304 2305 /** 2306 * Verify the external Rx Queue list is empty. 2307 * 2308 * @param dev 2309 * Pointer to Ethernet device. 2310 * 2311 * @return 2312 * The number of object not released. 2313 */ 2314 int 2315 mlx5_ext_rxq_verify(struct rte_eth_dev *dev) 2316 { 2317 struct mlx5_priv *priv = dev->data->dev_private; 2318 struct mlx5_external_rxq *rxq; 2319 uint32_t i; 2320 int ret = 0; 2321 2322 if (priv->ext_rxqs == NULL) 2323 return 0; 2324 2325 for (i = MLX5_EXTERNAL_RX_QUEUE_ID_MIN; i <= UINT16_MAX ; ++i) { 2326 rxq = mlx5_ext_rxq_get(dev, i); 2327 if (rxq->refcnt < 2) 2328 continue; 2329 DRV_LOG(DEBUG, "Port %u external RxQ %u still referenced.", 2330 dev->data->port_id, i); 2331 ++ret; 2332 } 2333 return ret; 2334 } 2335 2336 /** 2337 * Check whether RxQ type is Hairpin. 2338 * 2339 * @param dev 2340 * Pointer to Ethernet device. 2341 * @param idx 2342 * Rx queue index. 2343 * 2344 * @return 2345 * True if Rx queue type is Hairpin, otherwise False. 2346 */ 2347 bool 2348 mlx5_rxq_is_hairpin(struct rte_eth_dev *dev, uint16_t idx) 2349 { 2350 struct mlx5_rxq_ctrl *rxq_ctrl; 2351 2352 if (mlx5_is_external_rxq(dev, idx)) 2353 return false; 2354 rxq_ctrl = mlx5_rxq_ctrl_get(dev, idx); 2355 return (rxq_ctrl != NULL && rxq_ctrl->is_hairpin); 2356 } 2357 2358 /* 2359 * Get a Rx hairpin queue configuration. 2360 * 2361 * @param dev 2362 * Pointer to Ethernet device. 2363 * @param idx 2364 * Rx queue index. 2365 * 2366 * @return 2367 * Pointer to the configuration if a hairpin RX queue, otherwise NULL. 2368 */ 2369 const struct rte_eth_hairpin_conf * 2370 mlx5_rxq_get_hairpin_conf(struct rte_eth_dev *dev, uint16_t idx) 2371 { 2372 if (mlx5_rxq_is_hairpin(dev, idx)) { 2373 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, idx); 2374 2375 return rxq != NULL ? &rxq->hairpin_conf : NULL; 2376 } 2377 return NULL; 2378 } 2379 2380 /** 2381 * Match queues listed in arguments to queues contained in indirection table 2382 * object. 2383 * 2384 * @param ind_tbl 2385 * Pointer to indirection table to match. 2386 * @param queues 2387 * Queues to match to queues in indirection table. 2388 * @param queues_n 2389 * Number of queues in the array. 2390 * 2391 * @return 2392 * 1 if all queues in indirection table match 0 otherwise. 2393 */ 2394 static int 2395 mlx5_ind_table_obj_match_queues(const struct mlx5_ind_table_obj *ind_tbl, 2396 const uint16_t *queues, uint32_t queues_n) 2397 { 2398 return (ind_tbl->queues_n == queues_n) && 2399 (!memcmp(ind_tbl->queues, queues, 2400 ind_tbl->queues_n * sizeof(ind_tbl->queues[0]))); 2401 } 2402 2403 /** 2404 * Get an indirection table. 2405 * 2406 * @param dev 2407 * Pointer to Ethernet device. 2408 * @param queues 2409 * Queues entering in the indirection table. 2410 * @param queues_n 2411 * Number of queues in the array. 2412 * 2413 * @return 2414 * An indirection table if found. 2415 */ 2416 struct mlx5_ind_table_obj * 2417 mlx5_ind_table_obj_get(struct rte_eth_dev *dev, const uint16_t *queues, 2418 uint32_t queues_n) 2419 { 2420 struct mlx5_priv *priv = dev->data->dev_private; 2421 struct mlx5_ind_table_obj *ind_tbl; 2422 2423 rte_rwlock_read_lock(&priv->ind_tbls_lock); 2424 LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { 2425 if ((ind_tbl->queues_n == queues_n) && 2426 (memcmp(ind_tbl->queues, queues, 2427 ind_tbl->queues_n * sizeof(ind_tbl->queues[0])) 2428 == 0)) { 2429 __atomic_fetch_add(&ind_tbl->refcnt, 1, 2430 __ATOMIC_RELAXED); 2431 break; 2432 } 2433 } 2434 rte_rwlock_read_unlock(&priv->ind_tbls_lock); 2435 return ind_tbl; 2436 } 2437 2438 /** 2439 * Release an indirection table. 2440 * 2441 * @param dev 2442 * Pointer to Ethernet device. 2443 * @param ind_table 2444 * Indirection table to release. 2445 * @param deref_rxqs 2446 * If true, then dereference RX queues related to indirection table. 2447 * Otherwise, no additional action will be taken. 2448 * 2449 * @return 2450 * 1 while a reference on it exists, 0 when freed. 2451 */ 2452 int 2453 mlx5_ind_table_obj_release(struct rte_eth_dev *dev, 2454 struct mlx5_ind_table_obj *ind_tbl, 2455 bool deref_rxqs) 2456 { 2457 struct mlx5_priv *priv = dev->data->dev_private; 2458 unsigned int ret; 2459 2460 rte_rwlock_write_lock(&priv->ind_tbls_lock); 2461 ret = __atomic_sub_fetch(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED); 2462 if (!ret) 2463 LIST_REMOVE(ind_tbl, next); 2464 rte_rwlock_write_unlock(&priv->ind_tbls_lock); 2465 if (ret) 2466 return 1; 2467 priv->obj_ops.ind_table_destroy(ind_tbl); 2468 if (deref_rxqs) 2469 mlx5_rxqs_deref(dev, ind_tbl->queues, ind_tbl->queues_n); 2470 mlx5_free(ind_tbl); 2471 return 0; 2472 } 2473 2474 /** 2475 * Verify the Rx Queue list is empty 2476 * 2477 * @param dev 2478 * Pointer to Ethernet device. 2479 * 2480 * @return 2481 * The number of object not released. 2482 */ 2483 int 2484 mlx5_ind_table_obj_verify(struct rte_eth_dev *dev) 2485 { 2486 struct mlx5_priv *priv = dev->data->dev_private; 2487 struct mlx5_ind_table_obj *ind_tbl; 2488 int ret = 0; 2489 2490 rte_rwlock_read_lock(&priv->ind_tbls_lock); 2491 LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { 2492 DRV_LOG(DEBUG, 2493 "port %u indirection table obj %p still referenced", 2494 dev->data->port_id, (void *)ind_tbl); 2495 ++ret; 2496 } 2497 rte_rwlock_read_unlock(&priv->ind_tbls_lock); 2498 return ret; 2499 } 2500 2501 /** 2502 * Setup an indirection table structure fields. 2503 * 2504 * @param dev 2505 * Pointer to Ethernet device. 2506 * @param ind_table 2507 * Indirection table to modify. 2508 * @param ref_qs 2509 * Whether to increment RxQ reference counters. 2510 * 2511 * @return 2512 * 0 on success, a negative errno value otherwise and rte_errno is set. 2513 */ 2514 int 2515 mlx5_ind_table_obj_setup(struct rte_eth_dev *dev, 2516 struct mlx5_ind_table_obj *ind_tbl, 2517 bool ref_qs) 2518 { 2519 struct mlx5_priv *priv = dev->data->dev_private; 2520 uint32_t queues_n = ind_tbl->queues_n; 2521 int ret; 2522 const unsigned int n = rte_is_power_of_2(queues_n) ? 2523 log2above(queues_n) : 2524 log2above(priv->sh->dev_cap.ind_table_max_size); 2525 2526 if (ref_qs && mlx5_rxqs_ref(dev, ind_tbl->queues, queues_n) < 0) { 2527 DRV_LOG(DEBUG, "Port %u invalid indirection table queues.", 2528 dev->data->port_id); 2529 return -rte_errno; 2530 } 2531 ret = priv->obj_ops.ind_table_new(dev, n, ind_tbl); 2532 if (ret) { 2533 DRV_LOG(DEBUG, "Port %u cannot create a new indirection table.", 2534 dev->data->port_id); 2535 if (ref_qs) { 2536 int err = rte_errno; 2537 2538 mlx5_rxqs_deref(dev, ind_tbl->queues, queues_n); 2539 rte_errno = err; 2540 } 2541 return ret; 2542 } 2543 __atomic_fetch_add(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED); 2544 return 0; 2545 } 2546 2547 /** 2548 * Create an indirection table. 2549 * 2550 * @param dev 2551 * Pointer to Ethernet device. 2552 * @param queues 2553 * Queues entering in the indirection table. 2554 * @param queues_n 2555 * Number of queues in the array. 2556 * @param standalone 2557 * Indirection table for Standalone queue. 2558 * @param ref_qs 2559 * Whether to increment RxQ reference counters. 2560 * 2561 * @return 2562 * The Verbs/DevX object initialized, NULL otherwise and rte_errno is set. 2563 */ 2564 struct mlx5_ind_table_obj * 2565 mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues, 2566 uint32_t queues_n, bool standalone, bool ref_qs) 2567 { 2568 struct mlx5_priv *priv = dev->data->dev_private; 2569 struct mlx5_ind_table_obj *ind_tbl; 2570 int ret; 2571 2572 /* 2573 * Allocate maximum queues for shared action as queue number 2574 * maybe modified later. 2575 */ 2576 ind_tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*ind_tbl) + 2577 (standalone ? priv->rxqs_n : queues_n) * 2578 sizeof(uint16_t), 0, SOCKET_ID_ANY); 2579 if (!ind_tbl) { 2580 rte_errno = ENOMEM; 2581 return NULL; 2582 } 2583 ind_tbl->queues_n = queues_n; 2584 ind_tbl->queues = (uint16_t *)(ind_tbl + 1); 2585 memcpy(ind_tbl->queues, queues, queues_n * sizeof(*queues)); 2586 ret = mlx5_ind_table_obj_setup(dev, ind_tbl, ref_qs); 2587 if (ret < 0) { 2588 mlx5_free(ind_tbl); 2589 return NULL; 2590 } 2591 rte_rwlock_write_lock(&priv->ind_tbls_lock); 2592 if (!standalone) 2593 LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next); 2594 else 2595 LIST_INSERT_HEAD(&priv->standalone_ind_tbls, ind_tbl, next); 2596 rte_rwlock_write_unlock(&priv->ind_tbls_lock); 2597 2598 return ind_tbl; 2599 } 2600 2601 static int 2602 mlx5_ind_table_obj_check_standalone(struct rte_eth_dev *dev __rte_unused, 2603 struct mlx5_ind_table_obj *ind_tbl) 2604 { 2605 uint32_t refcnt; 2606 2607 refcnt = __atomic_load_n(&ind_tbl->refcnt, __ATOMIC_RELAXED); 2608 if (refcnt <= 1) 2609 return 0; 2610 /* 2611 * Modification of indirection tables having more than 1 2612 * reference is unsupported. 2613 */ 2614 DRV_LOG(DEBUG, 2615 "Port %u cannot modify indirection table %p (refcnt %u > 1).", 2616 dev->data->port_id, (void *)ind_tbl, refcnt); 2617 rte_errno = EINVAL; 2618 return -rte_errno; 2619 } 2620 2621 /** 2622 * Modify an indirection table. 2623 * 2624 * @param dev 2625 * Pointer to Ethernet device. 2626 * @param ind_table 2627 * Indirection table to modify. 2628 * @param queues 2629 * Queues replacement for the indirection table. 2630 * @param queues_n 2631 * Number of queues in the array. 2632 * @param standalone 2633 * Indirection table for Standalone queue. 2634 * @param ref_new_qs 2635 * Whether to increment new RxQ set reference counters. 2636 * @param deref_old_qs 2637 * Whether to decrement old RxQ set reference counters. 2638 * 2639 * @return 2640 * 0 on success, a negative errno value otherwise and rte_errno is set. 2641 */ 2642 int 2643 mlx5_ind_table_obj_modify(struct rte_eth_dev *dev, 2644 struct mlx5_ind_table_obj *ind_tbl, 2645 uint16_t *queues, const uint32_t queues_n, 2646 bool standalone, bool ref_new_qs, bool deref_old_qs) 2647 { 2648 struct mlx5_priv *priv = dev->data->dev_private; 2649 int ret; 2650 const unsigned int n = rte_is_power_of_2(queues_n) ? 2651 log2above(queues_n) : 2652 log2above(priv->sh->dev_cap.ind_table_max_size); 2653 2654 MLX5_ASSERT(standalone); 2655 RTE_SET_USED(standalone); 2656 if (mlx5_ind_table_obj_check_standalone(dev, ind_tbl) < 0) 2657 return -rte_errno; 2658 if (ref_new_qs && mlx5_rxqs_ref(dev, queues, queues_n) < 0) { 2659 DRV_LOG(DEBUG, "Port %u invalid indirection table queues.", 2660 dev->data->port_id); 2661 return -rte_errno; 2662 } 2663 MLX5_ASSERT(priv->obj_ops.ind_table_modify); 2664 ret = priv->obj_ops.ind_table_modify(dev, n, queues, queues_n, ind_tbl); 2665 if (ret) { 2666 DRV_LOG(DEBUG, "Port %u cannot modify indirection table.", 2667 dev->data->port_id); 2668 if (ref_new_qs) { 2669 int err = rte_errno; 2670 2671 mlx5_rxqs_deref(dev, queues, queues_n); 2672 rte_errno = err; 2673 } 2674 return ret; 2675 } 2676 if (deref_old_qs) 2677 mlx5_rxqs_deref(dev, ind_tbl->queues, ind_tbl->queues_n); 2678 ind_tbl->queues_n = queues_n; 2679 ind_tbl->queues = queues; 2680 return 0; 2681 } 2682 2683 /** 2684 * Attach an indirection table to its queues. 2685 * 2686 * @param dev 2687 * Pointer to Ethernet device. 2688 * @param ind_table 2689 * Indirection table to attach. 2690 * 2691 * @return 2692 * 0 on success, a negative errno value otherwise and rte_errno is set. 2693 */ 2694 int 2695 mlx5_ind_table_obj_attach(struct rte_eth_dev *dev, 2696 struct mlx5_ind_table_obj *ind_tbl) 2697 { 2698 int ret; 2699 2700 ret = mlx5_ind_table_obj_modify(dev, ind_tbl, ind_tbl->queues, 2701 ind_tbl->queues_n, 2702 true /* standalone */, 2703 true /* ref_new_qs */, 2704 false /* deref_old_qs */); 2705 if (ret != 0) 2706 DRV_LOG(ERR, "Port %u could not modify indirect table obj %p", 2707 dev->data->port_id, (void *)ind_tbl); 2708 return ret; 2709 } 2710 2711 /** 2712 * Detach an indirection table from its queues. 2713 * 2714 * @param dev 2715 * Pointer to Ethernet device. 2716 * @param ind_table 2717 * Indirection table to detach. 2718 * 2719 * @return 2720 * 0 on success, a negative errno value otherwise and rte_errno is set. 2721 */ 2722 int 2723 mlx5_ind_table_obj_detach(struct rte_eth_dev *dev, 2724 struct mlx5_ind_table_obj *ind_tbl) 2725 { 2726 struct mlx5_priv *priv = dev->data->dev_private; 2727 const unsigned int n = rte_is_power_of_2(ind_tbl->queues_n) ? 2728 log2above(ind_tbl->queues_n) : 2729 log2above(priv->sh->dev_cap.ind_table_max_size); 2730 unsigned int i; 2731 int ret; 2732 2733 ret = mlx5_ind_table_obj_check_standalone(dev, ind_tbl); 2734 if (ret != 0) 2735 return ret; 2736 MLX5_ASSERT(priv->obj_ops.ind_table_modify); 2737 ret = priv->obj_ops.ind_table_modify(dev, n, NULL, 0, ind_tbl); 2738 if (ret != 0) { 2739 DRV_LOG(ERR, "Port %u could not modify indirect table obj %p", 2740 dev->data->port_id, (void *)ind_tbl); 2741 return ret; 2742 } 2743 for (i = 0; i < ind_tbl->queues_n; i++) 2744 mlx5_rxq_release(dev, ind_tbl->queues[i]); 2745 return ret; 2746 } 2747 2748 int 2749 mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry, 2750 void *cb_ctx) 2751 { 2752 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 2753 struct mlx5_flow_rss_desc *rss_desc = ctx->data; 2754 struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry); 2755 2756 return (hrxq->rss_key_len != rss_desc->key_len || 2757 memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) || 2758 hrxq->hws_flags != rss_desc->hws_flags || 2759 hrxq->hash_fields != rss_desc->hash_fields || 2760 hrxq->ind_table->queues_n != rss_desc->queue_num || 2761 memcmp(hrxq->ind_table->queues, rss_desc->queue, 2762 rss_desc->queue_num * sizeof(rss_desc->queue[0]))); 2763 } 2764 2765 /** 2766 * Modify an Rx Hash queue configuration. 2767 * 2768 * @param dev 2769 * Pointer to Ethernet device. 2770 * @param hrxq 2771 * Index to Hash Rx queue to modify. 2772 * @param rss_key 2773 * RSS key for the Rx hash queue. 2774 * @param rss_key_len 2775 * RSS key length. 2776 * @param hash_fields 2777 * Verbs protocol hash field to make the RSS on. 2778 * @param queues 2779 * Queues entering in hash queue. In case of empty hash_fields only the 2780 * first queue index will be taken for the indirection table. 2781 * @param queues_n 2782 * Number of queues. 2783 * 2784 * @return 2785 * 0 on success, a negative errno value otherwise and rte_errno is set. 2786 */ 2787 int 2788 mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hrxq_idx, 2789 const uint8_t *rss_key, uint32_t rss_key_len, 2790 uint64_t hash_fields, 2791 const uint16_t *queues, uint32_t queues_n) 2792 { 2793 int err; 2794 struct mlx5_ind_table_obj *ind_tbl = NULL; 2795 struct mlx5_priv *priv = dev->data->dev_private; 2796 struct mlx5_hrxq *hrxq = 2797 mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); 2798 bool dev_started = !!dev->data->dev_started; 2799 int ret; 2800 2801 if (!hrxq) { 2802 rte_errno = EINVAL; 2803 return -rte_errno; 2804 } 2805 /* validations */ 2806 if (hrxq->rss_key_len != rss_key_len) { 2807 /* rss_key_len is fixed size 40 byte & not supposed to change */ 2808 rte_errno = EINVAL; 2809 return -rte_errno; 2810 } 2811 queues_n = hash_fields ? queues_n : 1; 2812 if (mlx5_ind_table_obj_match_queues(hrxq->ind_table, 2813 queues, queues_n)) { 2814 ind_tbl = hrxq->ind_table; 2815 } else { 2816 if (hrxq->standalone) { 2817 /* 2818 * Replacement of indirection table unsupported for 2819 * standalone hrxq objects (used by shared RSS). 2820 */ 2821 rte_errno = ENOTSUP; 2822 return -rte_errno; 2823 } 2824 ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n); 2825 if (!ind_tbl) 2826 ind_tbl = mlx5_ind_table_obj_new(dev, queues, queues_n, 2827 hrxq->standalone, 2828 dev_started); 2829 } 2830 if (!ind_tbl) { 2831 rte_errno = ENOMEM; 2832 return -rte_errno; 2833 } 2834 MLX5_ASSERT(priv->obj_ops.hrxq_modify); 2835 ret = priv->obj_ops.hrxq_modify(dev, hrxq, rss_key, 2836 hash_fields, ind_tbl); 2837 if (ret) { 2838 rte_errno = errno; 2839 goto error; 2840 } 2841 if (ind_tbl != hrxq->ind_table) { 2842 MLX5_ASSERT(!hrxq->standalone); 2843 mlx5_ind_table_obj_release(dev, hrxq->ind_table, true); 2844 hrxq->ind_table = ind_tbl; 2845 } 2846 hrxq->hash_fields = hash_fields; 2847 memcpy(hrxq->rss_key, rss_key, rss_key_len); 2848 return 0; 2849 error: 2850 err = rte_errno; 2851 if (ind_tbl != hrxq->ind_table) { 2852 MLX5_ASSERT(!hrxq->standalone); 2853 mlx5_ind_table_obj_release(dev, ind_tbl, true); 2854 } 2855 rte_errno = err; 2856 return -rte_errno; 2857 } 2858 2859 static void 2860 __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) 2861 { 2862 struct mlx5_priv *priv = dev->data->dev_private; 2863 2864 #ifdef HAVE_IBV_FLOW_DV_SUPPORT 2865 if (hrxq->hws_flags) 2866 mlx5dr_action_destroy(hrxq->action); 2867 else 2868 mlx5_glue->destroy_flow_action(hrxq->action); 2869 #endif 2870 priv->obj_ops.hrxq_destroy(hrxq); 2871 if (!hrxq->standalone) { 2872 mlx5_ind_table_obj_release(dev, hrxq->ind_table, 2873 hrxq->hws_flags ? 2874 (!!dev->data->dev_started) : true); 2875 } 2876 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx); 2877 } 2878 2879 /** 2880 * Release the hash Rx queue. 2881 * 2882 * @param dev 2883 * Pointer to Ethernet device. 2884 * @param hrxq 2885 * Index to Hash Rx queue to release. 2886 * 2887 * @param list 2888 * mlx5 list pointer. 2889 * @param entry 2890 * Hash queue entry pointer. 2891 */ 2892 void 2893 mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry) 2894 { 2895 struct rte_eth_dev *dev = tool_ctx; 2896 struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry); 2897 2898 __mlx5_hrxq_remove(dev, hrxq); 2899 } 2900 2901 static struct mlx5_hrxq * 2902 __mlx5_hrxq_create(struct rte_eth_dev *dev, 2903 struct mlx5_flow_rss_desc *rss_desc) 2904 { 2905 struct mlx5_priv *priv = dev->data->dev_private; 2906 const uint8_t *rss_key = rss_desc->key; 2907 uint32_t rss_key_len = rss_desc->key_len; 2908 bool standalone = !!rss_desc->shared_rss; 2909 const uint16_t *queues = 2910 standalone ? rss_desc->const_q : rss_desc->queue; 2911 uint32_t queues_n = rss_desc->queue_num; 2912 struct mlx5_hrxq *hrxq = NULL; 2913 uint32_t hrxq_idx = 0; 2914 struct mlx5_ind_table_obj *ind_tbl = rss_desc->ind_tbl; 2915 int ret; 2916 2917 queues_n = rss_desc->hash_fields ? queues_n : 1; 2918 if (!ind_tbl && !rss_desc->hws_flags) 2919 ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n); 2920 if (!ind_tbl) 2921 ind_tbl = mlx5_ind_table_obj_new(dev, queues, queues_n, 2922 standalone || 2923 rss_desc->hws_flags, 2924 !!dev->data->dev_started); 2925 if (!ind_tbl) 2926 return NULL; 2927 hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx); 2928 if (!hrxq) 2929 goto error; 2930 hrxq->standalone = standalone; 2931 hrxq->idx = hrxq_idx; 2932 hrxq->ind_table = ind_tbl; 2933 hrxq->rss_key_len = rss_key_len; 2934 hrxq->hash_fields = rss_desc->hash_fields; 2935 hrxq->hws_flags = rss_desc->hws_flags; 2936 memcpy(hrxq->rss_key, rss_key, rss_key_len); 2937 ret = priv->obj_ops.hrxq_new(dev, hrxq, rss_desc->tunnel); 2938 if (ret < 0) 2939 goto error; 2940 return hrxq; 2941 error: 2942 if (!rss_desc->ind_tbl) 2943 mlx5_ind_table_obj_release(dev, ind_tbl, true); 2944 if (hrxq) 2945 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); 2946 return NULL; 2947 } 2948 2949 struct mlx5_list_entry * 2950 mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx) 2951 { 2952 struct rte_eth_dev *dev = tool_ctx; 2953 struct mlx5_flow_cb_ctx *ctx = cb_ctx; 2954 struct mlx5_flow_rss_desc *rss_desc = ctx->data; 2955 struct mlx5_hrxq *hrxq; 2956 2957 hrxq = __mlx5_hrxq_create(dev, rss_desc); 2958 return hrxq ? &hrxq->entry : NULL; 2959 } 2960 2961 struct mlx5_list_entry * 2962 mlx5_hrxq_clone_cb(void *tool_ctx, struct mlx5_list_entry *entry, 2963 void *cb_ctx __rte_unused) 2964 { 2965 struct rte_eth_dev *dev = tool_ctx; 2966 struct mlx5_priv *priv = dev->data->dev_private; 2967 struct mlx5_hrxq *hrxq; 2968 uint32_t hrxq_idx = 0; 2969 2970 hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx); 2971 if (!hrxq) 2972 return NULL; 2973 memcpy(hrxq, entry, sizeof(*hrxq) + MLX5_RSS_HASH_KEY_LEN); 2974 hrxq->idx = hrxq_idx; 2975 return &hrxq->entry; 2976 } 2977 2978 void 2979 mlx5_hrxq_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry) 2980 { 2981 struct rte_eth_dev *dev = tool_ctx; 2982 struct mlx5_priv *priv = dev->data->dev_private; 2983 struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry); 2984 2985 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx); 2986 } 2987 2988 /** 2989 * Get an Rx Hash queue. 2990 * 2991 * @param dev 2992 * Pointer to Ethernet device. 2993 * @param rss_desc 2994 * RSS configuration for the Rx hash queue. 2995 * 2996 * @return 2997 * An hash Rx queue on success. 2998 */ 2999 struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, 3000 struct mlx5_flow_rss_desc *rss_desc) 3001 { 3002 struct mlx5_priv *priv = dev->data->dev_private; 3003 struct mlx5_hrxq *hrxq = NULL; 3004 struct mlx5_list_entry *entry; 3005 struct mlx5_flow_cb_ctx ctx = { 3006 .data = rss_desc, 3007 }; 3008 3009 if (rss_desc->shared_rss) { 3010 hrxq = __mlx5_hrxq_create(dev, rss_desc); 3011 } else { 3012 entry = mlx5_list_register(priv->hrxqs, &ctx); 3013 if (!entry) 3014 return NULL; 3015 hrxq = container_of(entry, typeof(*hrxq), entry); 3016 } 3017 return hrxq; 3018 } 3019 3020 /** 3021 * Release the hash Rx queue. 3022 * 3023 * @param dev 3024 * Pointer to Ethernet device. 3025 * @param hrxq_idx 3026 * Hash Rx queue to release. 3027 * 3028 * @return 3029 * 1 while a reference on it exists, 0 when freed. 3030 */ 3031 int mlx5_hrxq_obj_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) 3032 { 3033 struct mlx5_priv *priv = dev->data->dev_private; 3034 3035 if (!hrxq) 3036 return 0; 3037 if (!hrxq->standalone) 3038 return mlx5_list_unregister(priv->hrxqs, &hrxq->entry); 3039 __mlx5_hrxq_remove(dev, hrxq); 3040 return 0; 3041 } 3042 3043 /** 3044 * Release the hash Rx queue with index. 3045 * 3046 * @param dev 3047 * Pointer to Ethernet device. 3048 * @param hrxq_idx 3049 * Index to Hash Rx queue to release. 3050 * 3051 * @return 3052 * 1 while a reference on it exists, 0 when freed. 3053 */ 3054 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx) 3055 { 3056 struct mlx5_priv *priv = dev->data->dev_private; 3057 struct mlx5_hrxq *hrxq; 3058 3059 hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); 3060 return mlx5_hrxq_obj_release(dev, hrxq); 3061 } 3062 3063 /** 3064 * Create a drop Rx Hash queue. 3065 * 3066 * @param dev 3067 * Pointer to Ethernet device. 3068 * 3069 * @return 3070 * The Verbs/DevX object initialized, NULL otherwise and rte_errno is set. 3071 */ 3072 struct mlx5_hrxq * 3073 mlx5_drop_action_create(struct rte_eth_dev *dev) 3074 { 3075 struct mlx5_priv *priv = dev->data->dev_private; 3076 struct mlx5_hrxq *hrxq = NULL; 3077 int ret; 3078 3079 if (priv->drop_queue.hrxq) 3080 return priv->drop_queue.hrxq; 3081 hrxq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*hrxq), 0, SOCKET_ID_ANY); 3082 if (!hrxq) { 3083 DRV_LOG(WARNING, 3084 "Port %u cannot allocate memory for drop queue.", 3085 dev->data->port_id); 3086 rte_errno = ENOMEM; 3087 goto error; 3088 } 3089 priv->drop_queue.hrxq = hrxq; 3090 hrxq->ind_table = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*hrxq->ind_table), 3091 0, SOCKET_ID_ANY); 3092 if (!hrxq->ind_table) { 3093 rte_errno = ENOMEM; 3094 goto error; 3095 } 3096 ret = priv->obj_ops.drop_action_create(dev); 3097 if (ret < 0) 3098 goto error; 3099 return hrxq; 3100 error: 3101 if (hrxq) { 3102 if (hrxq->ind_table) 3103 mlx5_free(hrxq->ind_table); 3104 priv->drop_queue.hrxq = NULL; 3105 mlx5_free(hrxq); 3106 } 3107 return NULL; 3108 } 3109 3110 /** 3111 * Release a drop hash Rx queue. 3112 * 3113 * @param dev 3114 * Pointer to Ethernet device. 3115 */ 3116 void 3117 mlx5_drop_action_destroy(struct rte_eth_dev *dev) 3118 { 3119 struct mlx5_priv *priv = dev->data->dev_private; 3120 struct mlx5_hrxq *hrxq = priv->drop_queue.hrxq; 3121 3122 if (!priv->drop_queue.hrxq) 3123 return; 3124 priv->obj_ops.drop_action_destroy(dev); 3125 mlx5_free(priv->drop_queue.rxq); 3126 mlx5_free(hrxq->ind_table); 3127 mlx5_free(hrxq); 3128 priv->drop_queue.rxq = NULL; 3129 priv->drop_queue.hrxq = NULL; 3130 } 3131 3132 /** 3133 * Verify the Rx Queue list is empty 3134 * 3135 * @param dev 3136 * Pointer to Ethernet device. 3137 * 3138 * @return 3139 * The number of object not released. 3140 */ 3141 uint32_t 3142 mlx5_hrxq_verify(struct rte_eth_dev *dev) 3143 { 3144 struct mlx5_priv *priv = dev->data->dev_private; 3145 3146 return mlx5_list_get_entry_num(priv->hrxqs); 3147 } 3148 3149 /** 3150 * Set the Rx queue timestamp conversion parameters 3151 * 3152 * @param[in] dev 3153 * Pointer to the Ethernet device structure. 3154 */ 3155 void 3156 mlx5_rxq_timestamp_set(struct rte_eth_dev *dev) 3157 { 3158 struct mlx5_priv *priv = dev->data->dev_private; 3159 struct mlx5_dev_ctx_shared *sh = priv->sh; 3160 unsigned int i; 3161 3162 for (i = 0; i != priv->rxqs_n; ++i) { 3163 struct mlx5_rxq_data *data = mlx5_rxq_data_get(dev, i); 3164 3165 if (data == NULL) 3166 continue; 3167 data->sh = sh; 3168 data->rt_timestamp = sh->dev_cap.rt_timestamp; 3169 } 3170 } 3171 3172 /** 3173 * Validate given external RxQ rte_plow index, and get pointer to concurrent 3174 * external RxQ object to map/unmap. 3175 * 3176 * @param[in] port_id 3177 * The port identifier of the Ethernet device. 3178 * @param[in] dpdk_idx 3179 * Queue index in rte_flow. 3180 * 3181 * @return 3182 * Pointer to concurrent external RxQ on success, 3183 * NULL otherwise and rte_errno is set. 3184 */ 3185 static struct mlx5_external_rxq * 3186 mlx5_external_rx_queue_get_validate(uint16_t port_id, uint16_t dpdk_idx) 3187 { 3188 struct rte_eth_dev *dev; 3189 struct mlx5_priv *priv; 3190 3191 if (dpdk_idx < MLX5_EXTERNAL_RX_QUEUE_ID_MIN) { 3192 DRV_LOG(ERR, "Queue index %u should be in range: [%u, %u].", 3193 dpdk_idx, MLX5_EXTERNAL_RX_QUEUE_ID_MIN, UINT16_MAX); 3194 rte_errno = EINVAL; 3195 return NULL; 3196 } 3197 if (rte_eth_dev_is_valid_port(port_id) < 0) { 3198 DRV_LOG(ERR, "There is no Ethernet device for port %u.", 3199 port_id); 3200 rte_errno = ENODEV; 3201 return NULL; 3202 } 3203 dev = &rte_eth_devices[port_id]; 3204 priv = dev->data->dev_private; 3205 if (!mlx5_imported_pd_and_ctx(priv->sh->cdev)) { 3206 DRV_LOG(ERR, "Port %u " 3207 "external RxQ isn't supported on local PD and CTX.", 3208 port_id); 3209 rte_errno = ENOTSUP; 3210 return NULL; 3211 } 3212 if (!mlx5_devx_obj_ops_en(priv->sh)) { 3213 DRV_LOG(ERR, 3214 "Port %u external RxQ isn't supported by Verbs API.", 3215 port_id); 3216 rte_errno = ENOTSUP; 3217 return NULL; 3218 } 3219 /* 3220 * When user configures remote PD and CTX and device creates RxQ by 3221 * DevX, external RxQs array is allocated. 3222 */ 3223 MLX5_ASSERT(priv->ext_rxqs != NULL); 3224 return &priv->ext_rxqs[dpdk_idx - MLX5_EXTERNAL_RX_QUEUE_ID_MIN]; 3225 } 3226 3227 int 3228 rte_pmd_mlx5_external_rx_queue_id_map(uint16_t port_id, uint16_t dpdk_idx, 3229 uint32_t hw_idx) 3230 { 3231 struct mlx5_external_rxq *ext_rxq; 3232 uint32_t unmapped = 0; 3233 3234 ext_rxq = mlx5_external_rx_queue_get_validate(port_id, dpdk_idx); 3235 if (ext_rxq == NULL) 3236 return -rte_errno; 3237 if (!__atomic_compare_exchange_n(&ext_rxq->refcnt, &unmapped, 1, false, 3238 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { 3239 if (ext_rxq->hw_id != hw_idx) { 3240 DRV_LOG(ERR, "Port %u external RxQ index %u " 3241 "is already mapped to HW index (requesting is " 3242 "%u, existing is %u).", 3243 port_id, dpdk_idx, hw_idx, ext_rxq->hw_id); 3244 rte_errno = EEXIST; 3245 return -rte_errno; 3246 } 3247 DRV_LOG(WARNING, "Port %u external RxQ index %u " 3248 "is already mapped to the requested HW index (%u)", 3249 port_id, dpdk_idx, hw_idx); 3250 3251 } else { 3252 ext_rxq->hw_id = hw_idx; 3253 DRV_LOG(DEBUG, "Port %u external RxQ index %u " 3254 "is successfully mapped to the requested HW index (%u)", 3255 port_id, dpdk_idx, hw_idx); 3256 } 3257 return 0; 3258 } 3259 3260 int 3261 rte_pmd_mlx5_external_rx_queue_id_unmap(uint16_t port_id, uint16_t dpdk_idx) 3262 { 3263 struct mlx5_external_rxq *ext_rxq; 3264 uint32_t mapped = 1; 3265 3266 ext_rxq = mlx5_external_rx_queue_get_validate(port_id, dpdk_idx); 3267 if (ext_rxq == NULL) 3268 return -rte_errno; 3269 if (ext_rxq->refcnt > 1) { 3270 DRV_LOG(ERR, "Port %u external RxQ index %u still referenced.", 3271 port_id, dpdk_idx); 3272 rte_errno = EINVAL; 3273 return -rte_errno; 3274 } 3275 if (!__atomic_compare_exchange_n(&ext_rxq->refcnt, &mapped, 0, false, 3276 __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { 3277 DRV_LOG(ERR, "Port %u external RxQ index %u doesn't exist.", 3278 port_id, dpdk_idx); 3279 rte_errno = EINVAL; 3280 return -rte_errno; 3281 } 3282 DRV_LOG(DEBUG, 3283 "Port %u external RxQ index %u is successfully unmapped.", 3284 port_id, dpdk_idx); 3285 return 0; 3286 } 3287