1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2015 6WIND S.A. 5 * Copyright 2015 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stddef.h> 35 #include <assert.h> 36 #include <errno.h> 37 #include <string.h> 38 #include <stdint.h> 39 #include <unistd.h> 40 #include <sys/mman.h> 41 42 /* Verbs header. */ 43 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ 44 #ifdef PEDANTIC 45 #pragma GCC diagnostic ignored "-Wpedantic" 46 #endif 47 #include <infiniband/verbs.h> 48 #ifdef PEDANTIC 49 #pragma GCC diagnostic error "-Wpedantic" 50 #endif 51 52 #include <rte_mbuf.h> 53 #include <rte_malloc.h> 54 #include <rte_ethdev.h> 55 #include <rte_common.h> 56 57 #include "mlx5_utils.h" 58 #include "mlx5_defs.h" 59 #include "mlx5.h" 60 #include "mlx5_rxtx.h" 61 #include "mlx5_autoconf.h" 62 63 /** 64 * Allocate TX queue elements. 65 * 66 * @param txq_ctrl 67 * Pointer to TX queue structure. 68 */ 69 void 70 txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl) 71 { 72 const unsigned int elts_n = 1 << txq_ctrl->txq.elts_n; 73 unsigned int i; 74 75 for (i = 0; (i != elts_n); ++i) 76 (*txq_ctrl->txq.elts)[i] = NULL; 77 DEBUG("%p: allocated and configured %u WRs", (void *)txq_ctrl, elts_n); 78 txq_ctrl->txq.elts_head = 0; 79 txq_ctrl->txq.elts_tail = 0; 80 txq_ctrl->txq.elts_comp = 0; 81 } 82 83 /** 84 * Free TX queue elements. 85 * 86 * @param txq_ctrl 87 * Pointer to TX queue structure. 88 */ 89 static void 90 txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) 91 { 92 const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n; 93 const uint16_t elts_m = elts_n - 1; 94 uint16_t elts_head = txq_ctrl->txq.elts_head; 95 uint16_t elts_tail = txq_ctrl->txq.elts_tail; 96 struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts; 97 98 DEBUG("%p: freeing WRs", (void *)txq_ctrl); 99 txq_ctrl->txq.elts_head = 0; 100 txq_ctrl->txq.elts_tail = 0; 101 txq_ctrl->txq.elts_comp = 0; 102 103 while (elts_tail != elts_head) { 104 struct rte_mbuf *elt = (*elts)[elts_tail & elts_m]; 105 106 assert(elt != NULL); 107 rte_pktmbuf_free_seg(elt); 108 #ifndef NDEBUG 109 /* Poisoning. */ 110 memset(&(*elts)[elts_tail & elts_m], 111 0x77, 112 sizeof((*elts)[elts_tail & elts_m])); 113 #endif 114 ++elts_tail; 115 } 116 } 117 118 /** 119 * DPDK callback to configure a TX queue. 120 * 121 * @param dev 122 * Pointer to Ethernet device structure. 123 * @param idx 124 * TX queue index. 125 * @param desc 126 * Number of descriptors to configure in queue. 127 * @param socket 128 * NUMA socket on which memory must be allocated. 129 * @param[in] conf 130 * Thresholds parameters. 131 * 132 * @return 133 * 0 on success, negative errno value on failure. 134 */ 135 int 136 mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 137 unsigned int socket, const struct rte_eth_txconf *conf) 138 { 139 struct priv *priv = dev->data->dev_private; 140 struct mlx5_txq_data *txq = (*priv->txqs)[idx]; 141 struct mlx5_txq_ctrl *txq_ctrl = 142 container_of(txq, struct mlx5_txq_ctrl, txq); 143 int ret = 0; 144 145 priv_lock(priv); 146 if (desc <= MLX5_TX_COMP_THRESH) { 147 WARN("%p: number of descriptors requested for TX queue %u" 148 " must be higher than MLX5_TX_COMP_THRESH, using" 149 " %u instead of %u", 150 (void *)dev, idx, MLX5_TX_COMP_THRESH + 1, desc); 151 desc = MLX5_TX_COMP_THRESH + 1; 152 } 153 if (!rte_is_power_of_2(desc)) { 154 desc = 1 << log2above(desc); 155 WARN("%p: increased number of descriptors in TX queue %u" 156 " to the next power of two (%d)", 157 (void *)dev, idx, desc); 158 } 159 DEBUG("%p: configuring queue %u for %u descriptors", 160 (void *)dev, idx, desc); 161 if (idx >= priv->txqs_n) { 162 ERROR("%p: queue index out of range (%u >= %u)", 163 (void *)dev, idx, priv->txqs_n); 164 priv_unlock(priv); 165 return -EOVERFLOW; 166 } 167 if (!mlx5_priv_txq_releasable(priv, idx)) { 168 ret = EBUSY; 169 ERROR("%p: unable to release queue index %u", 170 (void *)dev, idx); 171 goto out; 172 } 173 mlx5_priv_txq_release(priv, idx); 174 txq_ctrl = mlx5_priv_txq_new(priv, idx, desc, socket, conf); 175 if (!txq_ctrl) { 176 ERROR("%p: unable to allocate queue index %u", 177 (void *)dev, idx); 178 ret = ENOMEM; 179 goto out; 180 } 181 DEBUG("%p: adding TX queue %p to list", 182 (void *)dev, (void *)txq_ctrl); 183 (*priv->txqs)[idx] = &txq_ctrl->txq; 184 out: 185 priv_unlock(priv); 186 return -ret; 187 } 188 189 /** 190 * DPDK callback to release a TX queue. 191 * 192 * @param dpdk_txq 193 * Generic TX queue pointer. 194 */ 195 void 196 mlx5_tx_queue_release(void *dpdk_txq) 197 { 198 struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; 199 struct mlx5_txq_ctrl *txq_ctrl; 200 struct priv *priv; 201 unsigned int i; 202 203 if (txq == NULL) 204 return; 205 txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); 206 priv = txq_ctrl->priv; 207 priv_lock(priv); 208 for (i = 0; (i != priv->txqs_n); ++i) 209 if ((*priv->txqs)[i] == txq) { 210 DEBUG("%p: removing TX queue %p from list", 211 (void *)priv->dev, (void *)txq_ctrl); 212 mlx5_priv_txq_release(priv, i); 213 break; 214 } 215 priv_unlock(priv); 216 } 217 218 219 /** 220 * Map locally UAR used in Tx queues for BlueFlame doorbell. 221 * 222 * @param[in] priv 223 * Pointer to private structure. 224 * @param fd 225 * Verbs file descriptor to map UAR pages. 226 * 227 * @return 228 * 0 on success, errno value on failure. 229 */ 230 int 231 priv_tx_uar_remap(struct priv *priv, int fd) 232 { 233 unsigned int i, j; 234 uintptr_t pages[priv->txqs_n]; 235 unsigned int pages_n = 0; 236 uintptr_t uar_va; 237 void *addr; 238 struct mlx5_txq_data *txq; 239 struct mlx5_txq_ctrl *txq_ctrl; 240 int already_mapped; 241 size_t page_size = sysconf(_SC_PAGESIZE); 242 243 memset(pages, 0, priv->txqs_n * sizeof(uintptr_t)); 244 /* 245 * As rdma-core, UARs are mapped in size of OS page size. 246 * Use aligned address to avoid duplicate mmap. 247 * Ref to libmlx5 function: mlx5_init_context() 248 */ 249 for (i = 0; i != priv->txqs_n; ++i) { 250 txq = (*priv->txqs)[i]; 251 txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); 252 uar_va = (uintptr_t)txq_ctrl->txq.bf_reg; 253 uar_va = RTE_ALIGN_FLOOR(uar_va, page_size); 254 already_mapped = 0; 255 for (j = 0; j != pages_n; ++j) { 256 if (pages[j] == uar_va) { 257 already_mapped = 1; 258 break; 259 } 260 } 261 if (already_mapped) 262 continue; 263 pages[pages_n++] = uar_va; 264 addr = mmap((void *)uar_va, page_size, 265 PROT_WRITE, MAP_FIXED | MAP_SHARED, fd, 266 txq_ctrl->uar_mmap_offset); 267 if (addr != (void *)uar_va) { 268 ERROR("call to mmap failed on UAR for txq %d\n", i); 269 return -1; 270 } 271 } 272 return 0; 273 } 274 275 /** 276 * Create the Tx queue Verbs object. 277 * 278 * @param priv 279 * Pointer to private structure. 280 * @param idx 281 * Queue index in DPDK Rx queue array 282 * 283 * @return 284 * The Verbs object initialised if it can be created. 285 */ 286 struct mlx5_txq_ibv* 287 mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) 288 { 289 struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; 290 struct mlx5_txq_ctrl *txq_ctrl = 291 container_of(txq_data, struct mlx5_txq_ctrl, txq); 292 struct mlx5_txq_ibv tmpl; 293 struct mlx5_txq_ibv *txq_ibv; 294 union { 295 struct ibv_qp_init_attr_ex init; 296 struct ibv_cq_init_attr_ex cq; 297 struct ibv_qp_attr mod; 298 struct ibv_cq_ex cq_attr; 299 } attr; 300 unsigned int cqe_n; 301 struct mlx5dv_qp qp = { .comp_mask = MLX5DV_QP_MASK_UAR_MMAP_OFFSET }; 302 struct mlx5dv_cq cq_info; 303 struct mlx5dv_obj obj; 304 const int desc = 1 << txq_data->elts_n; 305 int ret = 0; 306 307 assert(txq_data); 308 if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { 309 ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set"); 310 goto error; 311 } 312 memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv)); 313 /* MRs will be registered in mp2mr[] later. */ 314 attr.cq = (struct ibv_cq_init_attr_ex){ 315 .comp_mask = 0, 316 }; 317 cqe_n = ((desc / MLX5_TX_COMP_THRESH) - 1) ? 318 ((desc / MLX5_TX_COMP_THRESH) - 1) : 1; 319 if (priv->mps == MLX5_MPW_ENHANCED) 320 cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV; 321 tmpl.cq = ibv_create_cq(priv->ctx, cqe_n, NULL, NULL, 0); 322 if (tmpl.cq == NULL) { 323 ERROR("%p: CQ creation failure", (void *)txq_ctrl); 324 goto error; 325 } 326 attr.init = (struct ibv_qp_init_attr_ex){ 327 /* CQ to be associated with the send queue. */ 328 .send_cq = tmpl.cq, 329 /* CQ to be associated with the receive queue. */ 330 .recv_cq = tmpl.cq, 331 .cap = { 332 /* Max number of outstanding WRs. */ 333 .max_send_wr = 334 ((priv->device_attr.orig_attr.max_qp_wr < 335 desc) ? 336 priv->device_attr.orig_attr.max_qp_wr : 337 desc), 338 /* 339 * Max number of scatter/gather elements in a WR, 340 * must be 1 to prevent libmlx5 from trying to affect 341 * too much memory. TX gather is not impacted by the 342 * priv->device_attr.max_sge limit and will still work 343 * properly. 344 */ 345 .max_send_sge = 1, 346 }, 347 .qp_type = IBV_QPT_RAW_PACKET, 348 /* 349 * Do *NOT* enable this, completions events are managed per 350 * Tx burst. 351 */ 352 .sq_sig_all = 0, 353 .pd = priv->pd, 354 .comp_mask = IBV_QP_INIT_ATTR_PD, 355 }; 356 if (txq_data->max_inline) 357 attr.init.cap.max_inline_data = txq_ctrl->max_inline_data; 358 if (txq_data->tso_en) { 359 attr.init.max_tso_header = txq_ctrl->max_tso_header; 360 attr.init.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER; 361 } 362 tmpl.qp = ibv_create_qp_ex(priv->ctx, &attr.init); 363 if (tmpl.qp == NULL) { 364 ERROR("%p: QP creation failure", (void *)txq_ctrl); 365 goto error; 366 } 367 attr.mod = (struct ibv_qp_attr){ 368 /* Move the QP to this state. */ 369 .qp_state = IBV_QPS_INIT, 370 /* Primary port number. */ 371 .port_num = priv->port 372 }; 373 ret = ibv_modify_qp(tmpl.qp, &attr.mod, (IBV_QP_STATE | IBV_QP_PORT)); 374 if (ret) { 375 ERROR("%p: QP state to IBV_QPS_INIT failed", (void *)txq_ctrl); 376 goto error; 377 } 378 attr.mod = (struct ibv_qp_attr){ 379 .qp_state = IBV_QPS_RTR 380 }; 381 ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); 382 if (ret) { 383 ERROR("%p: QP state to IBV_QPS_RTR failed", (void *)txq_ctrl); 384 goto error; 385 } 386 attr.mod.qp_state = IBV_QPS_RTS; 387 ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); 388 if (ret) { 389 ERROR("%p: QP state to IBV_QPS_RTS failed", (void *)txq_ctrl); 390 goto error; 391 } 392 txq_ibv = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_txq_ibv), 0, 393 txq_ctrl->socket); 394 if (!txq_ibv) { 395 ERROR("%p: cannot allocate memory", (void *)txq_ctrl); 396 goto error; 397 } 398 obj.cq.in = tmpl.cq; 399 obj.cq.out = &cq_info; 400 obj.qp.in = tmpl.qp; 401 obj.qp.out = &qp; 402 ret = mlx5dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP); 403 if (ret != 0) 404 goto error; 405 if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) { 406 ERROR("Wrong MLX5_CQE_SIZE environment variable value: " 407 "it should be set to %u", RTE_CACHE_LINE_SIZE); 408 goto error; 409 } 410 txq_data->cqe_n = log2above(cq_info.cqe_cnt); 411 txq_data->qp_num_8s = tmpl.qp->qp_num << 8; 412 txq_data->wqes = qp.sq.buf; 413 txq_data->wqe_n = log2above(qp.sq.wqe_cnt); 414 txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR]; 415 txq_data->bf_reg = qp.bf.reg; 416 txq_data->cq_db = cq_info.dbrec; 417 txq_data->cqes = 418 (volatile struct mlx5_cqe (*)[]) 419 (uintptr_t)cq_info.buf; 420 txq_data->cq_ci = 0; 421 #ifndef NDEBUG 422 txq_data->cq_pi = 0; 423 #endif 424 txq_data->wqe_ci = 0; 425 txq_data->wqe_pi = 0; 426 txq_ibv->qp = tmpl.qp; 427 txq_ibv->cq = tmpl.cq; 428 rte_atomic32_inc(&txq_ibv->refcnt); 429 if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) { 430 txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset; 431 } else { 432 ERROR("Failed to retrieve UAR info, invalid libmlx5.so version"); 433 goto error; 434 } 435 DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, 436 (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); 437 LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next); 438 return txq_ibv; 439 error: 440 if (tmpl.cq) 441 claim_zero(ibv_destroy_cq(tmpl.cq)); 442 if (tmpl.qp) 443 claim_zero(ibv_destroy_qp(tmpl.qp)); 444 return NULL; 445 } 446 447 /** 448 * Get an Tx queue Verbs object. 449 * 450 * @param priv 451 * Pointer to private structure. 452 * @param idx 453 * Queue index in DPDK Rx queue array 454 * 455 * @return 456 * The Verbs object if it exists. 457 */ 458 struct mlx5_txq_ibv* 459 mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) 460 { 461 struct mlx5_txq_ctrl *txq_ctrl; 462 463 if (idx >= priv->txqs_n) 464 return NULL; 465 if (!(*priv->txqs)[idx]) 466 return NULL; 467 txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); 468 if (txq_ctrl->ibv) { 469 rte_atomic32_inc(&txq_ctrl->ibv->refcnt); 470 DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, 471 (void *)txq_ctrl->ibv, 472 rte_atomic32_read(&txq_ctrl->ibv->refcnt)); 473 } 474 return txq_ctrl->ibv; 475 } 476 477 /** 478 * Release an Tx verbs queue object. 479 * 480 * @param priv 481 * Pointer to private structure. 482 * @param txq_ibv 483 * Verbs Tx queue object. 484 * 485 * @return 486 * 0 on success, errno on failure. 487 */ 488 int 489 mlx5_priv_txq_ibv_release(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) 490 { 491 (void)priv; 492 assert(txq_ibv); 493 DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, 494 (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); 495 if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) { 496 claim_zero(ibv_destroy_qp(txq_ibv->qp)); 497 claim_zero(ibv_destroy_cq(txq_ibv->cq)); 498 LIST_REMOVE(txq_ibv, next); 499 rte_free(txq_ibv); 500 return 0; 501 } 502 return EBUSY; 503 } 504 505 /** 506 * Return true if a single reference exists on the object. 507 * 508 * @param priv 509 * Pointer to private structure. 510 * @param txq_ibv 511 * Verbs Tx queue object. 512 */ 513 int 514 mlx5_priv_txq_ibv_releasable(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) 515 { 516 (void)priv; 517 assert(txq_ibv); 518 return (rte_atomic32_read(&txq_ibv->refcnt) == 1); 519 } 520 521 /** 522 * Verify the Verbs Tx queue list is empty 523 * 524 * @param priv 525 * Pointer to private structure. 526 * 527 * @return the number of object not released. 528 */ 529 int 530 mlx5_priv_txq_ibv_verify(struct priv *priv) 531 { 532 int ret = 0; 533 struct mlx5_txq_ibv *txq_ibv; 534 535 LIST_FOREACH(txq_ibv, &priv->txqsibv, next) { 536 DEBUG("%p: Verbs Tx queue %p still referenced", (void *)priv, 537 (void *)txq_ibv); 538 ++ret; 539 } 540 return ret; 541 } 542 543 /** 544 * Create a DPDK Tx queue. 545 * 546 * @param priv 547 * Pointer to private structure. 548 * @param idx 549 * TX queue index. 550 * @param desc 551 * Number of descriptors to configure in queue. 552 * @param socket 553 * NUMA socket on which memory must be allocated. 554 * @param[in] conf 555 * Thresholds parameters. 556 * 557 * @return 558 * A DPDK queue object on success. 559 */ 560 struct mlx5_txq_ctrl* 561 mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, 562 unsigned int socket, 563 const struct rte_eth_txconf *conf) 564 { 565 const unsigned int max_tso_inline = 566 ((MLX5_MAX_TSO_HEADER + (RTE_CACHE_LINE_SIZE - 1)) / 567 RTE_CACHE_LINE_SIZE); 568 struct mlx5_txq_ctrl *tmpl; 569 570 tmpl = rte_calloc_socket("TXQ", 1, 571 sizeof(*tmpl) + 572 desc * sizeof(struct rte_mbuf *), 573 0, socket); 574 if (!tmpl) 575 return NULL; 576 assert(desc > MLX5_TX_COMP_THRESH); 577 tmpl->txq.flags = conf->txq_flags; 578 tmpl->priv = priv; 579 tmpl->socket = socket; 580 tmpl->txq.elts_n = log2above(desc); 581 if (priv->mps == MLX5_MPW_ENHANCED) 582 tmpl->txq.mpw_hdr_dseg = priv->mpw_hdr_dseg; 583 /* MRs will be registered in mp2mr[] later. */ 584 DEBUG("priv->device_attr.max_qp_wr is %d", 585 priv->device_attr.orig_attr.max_qp_wr); 586 DEBUG("priv->device_attr.max_sge is %d", 587 priv->device_attr.orig_attr.max_sge); 588 if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { 589 unsigned int ds_cnt; 590 591 tmpl->txq.max_inline = 592 ((priv->txq_inline + (RTE_CACHE_LINE_SIZE - 1)) / 593 RTE_CACHE_LINE_SIZE); 594 /* TSO and MPS can't be enabled concurrently. */ 595 assert(!priv->tso || !priv->mps); 596 if (priv->mps == MLX5_MPW_ENHANCED) { 597 tmpl->txq.inline_max_packet_sz = 598 priv->inline_max_packet_sz; 599 /* To minimize the size of data set, avoid requesting 600 * too large WQ. 601 */ 602 tmpl->max_inline_data = 603 ((RTE_MIN(priv->txq_inline, 604 priv->inline_max_packet_sz) + 605 (RTE_CACHE_LINE_SIZE - 1)) / 606 RTE_CACHE_LINE_SIZE) * RTE_CACHE_LINE_SIZE; 607 } else if (priv->tso) { 608 int inline_diff = tmpl->txq.max_inline - max_tso_inline; 609 610 /* 611 * Adjust inline value as Verbs aggregates 612 * tso_inline and txq_inline fields. 613 */ 614 tmpl->max_inline_data = inline_diff > 0 ? 615 inline_diff * 616 RTE_CACHE_LINE_SIZE : 617 0; 618 } else { 619 tmpl->max_inline_data = 620 tmpl->txq.max_inline * RTE_CACHE_LINE_SIZE; 621 } 622 /* 623 * Check if the inline size is too large in a way which 624 * can make the WQE DS to overflow. 625 * Considering in calculation: 626 * WQE CTRL (1 DS) 627 * WQE ETH (1 DS) 628 * Inline part (N DS) 629 */ 630 ds_cnt = 2 + (tmpl->txq.max_inline / MLX5_WQE_DWORD_SIZE); 631 if (ds_cnt > MLX5_DSEG_MAX) { 632 unsigned int max_inline = (MLX5_DSEG_MAX - 2) * 633 MLX5_WQE_DWORD_SIZE; 634 635 max_inline = max_inline - (max_inline % 636 RTE_CACHE_LINE_SIZE); 637 WARN("txq inline is too large (%d) setting it to " 638 "the maximum possible: %d\n", 639 priv->txq_inline, max_inline); 640 tmpl->txq.max_inline = max_inline / RTE_CACHE_LINE_SIZE; 641 } 642 } 643 if (priv->tso) { 644 tmpl->max_tso_header = max_tso_inline * RTE_CACHE_LINE_SIZE; 645 tmpl->txq.max_inline = RTE_MAX(tmpl->txq.max_inline, 646 max_tso_inline); 647 tmpl->txq.tso_en = 1; 648 } 649 if (priv->tunnel_en) 650 tmpl->txq.tunnel_en = 1; 651 tmpl->txq.elts = 652 (struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1); 653 tmpl->txq.stats.idx = idx; 654 rte_atomic32_inc(&tmpl->refcnt); 655 DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, 656 (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); 657 LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next); 658 return tmpl; 659 } 660 661 /** 662 * Get a Tx queue. 663 * 664 * @param priv 665 * Pointer to private structure. 666 * @param idx 667 * TX queue index. 668 * 669 * @return 670 * A pointer to the queue if it exists. 671 */ 672 struct mlx5_txq_ctrl* 673 mlx5_priv_txq_get(struct priv *priv, uint16_t idx) 674 { 675 struct mlx5_txq_ctrl *ctrl = NULL; 676 677 if ((*priv->txqs)[idx]) { 678 ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, 679 txq); 680 unsigned int i; 681 682 mlx5_priv_txq_ibv_get(priv, idx); 683 for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { 684 struct mlx5_mr *mr = NULL; 685 686 (void)mr; 687 if (ctrl->txq.mp2mr[i]) { 688 mr = priv_mr_get(priv, ctrl->txq.mp2mr[i]->mp); 689 assert(mr); 690 } 691 } 692 rte_atomic32_inc(&ctrl->refcnt); 693 DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, 694 (void *)ctrl, rte_atomic32_read(&ctrl->refcnt)); 695 } 696 return ctrl; 697 } 698 699 /** 700 * Release a Tx queue. 701 * 702 * @param priv 703 * Pointer to private structure. 704 * @param idx 705 * TX queue index. 706 * 707 * @return 708 * 0 on success, errno on failure. 709 */ 710 int 711 mlx5_priv_txq_release(struct priv *priv, uint16_t idx) 712 { 713 unsigned int i; 714 struct mlx5_txq_ctrl *txq; 715 716 if (!(*priv->txqs)[idx]) 717 return 0; 718 txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); 719 DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, 720 (void *)txq, rte_atomic32_read(&txq->refcnt)); 721 if (txq->ibv) { 722 int ret; 723 724 ret = mlx5_priv_txq_ibv_release(priv, txq->ibv); 725 if (!ret) 726 txq->ibv = NULL; 727 } 728 for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { 729 if (txq->txq.mp2mr[i]) { 730 priv_mr_release(priv, txq->txq.mp2mr[i]); 731 txq->txq.mp2mr[i] = NULL; 732 } 733 } 734 if (rte_atomic32_dec_and_test(&txq->refcnt)) { 735 txq_free_elts(txq); 736 LIST_REMOVE(txq, next); 737 rte_free(txq); 738 (*priv->txqs)[idx] = NULL; 739 return 0; 740 } 741 return EBUSY; 742 } 743 744 /** 745 * Verify if the queue can be released. 746 * 747 * @param priv 748 * Pointer to private structure. 749 * @param idx 750 * TX queue index. 751 * 752 * @return 753 * 1 if the queue can be released. 754 */ 755 int 756 mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx) 757 { 758 struct mlx5_txq_ctrl *txq; 759 760 if (!(*priv->txqs)[idx]) 761 return -1; 762 txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); 763 return (rte_atomic32_read(&txq->refcnt) == 1); 764 } 765 766 /** 767 * Verify the Tx Queue list is empty 768 * 769 * @param priv 770 * Pointer to private structure. 771 * 772 * @return the number of object not released. 773 */ 774 int 775 mlx5_priv_txq_verify(struct priv *priv) 776 { 777 struct mlx5_txq_ctrl *txq; 778 int ret = 0; 779 780 LIST_FOREACH(txq, &priv->txqsctrl, next) { 781 DEBUG("%p: Tx Queue %p still referenced", (void *)priv, 782 (void *)txq); 783 ++ret; 784 } 785 return ret; 786 } 787