1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2021 6WIND S.A. 3 * Copyright 2021 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdint.h> 7 #include <string.h> 8 #include <stdlib.h> 9 10 #include <rte_mbuf.h> 11 #include <rte_mempool.h> 12 #include <rte_prefetch.h> 13 #include <rte_common.h> 14 #include <rte_branch_prediction.h> 15 #include <rte_ether.h> 16 #include <rte_cycles.h> 17 #include <rte_flow.h> 18 19 #include <mlx5_prm.h> 20 #include <mlx5_common.h> 21 22 #include "mlx5_autoconf.h" 23 #include "mlx5_defs.h" 24 #include "mlx5.h" 25 #include "mlx5_utils.h" 26 #include "mlx5_rxtx.h" 27 #include "mlx5_tx.h" 28 29 #define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx}, 30 31 /** 32 * Move QP from error state to running state and initialize indexes. 33 * 34 * @param txq_ctrl 35 * Pointer to TX queue control structure. 36 * 37 * @return 38 * 0 on success, else -1. 39 */ 40 static int 41 tx_recover_qp(struct mlx5_txq_ctrl *txq_ctrl) 42 { 43 struct mlx5_mp_arg_queue_state_modify sm = { 44 .is_wq = 0, 45 .queue_id = txq_ctrl->txq.idx, 46 }; 47 48 if (mlx5_queue_state_modify(ETH_DEV(txq_ctrl->priv), &sm)) 49 return -1; 50 txq_ctrl->txq.wqe_ci = 0; 51 txq_ctrl->txq.wqe_pi = 0; 52 txq_ctrl->txq.elts_comp = 0; 53 return 0; 54 } 55 56 /* Return 1 if the error CQE is signed otherwise, sign it and return 0. */ 57 static int 58 check_err_cqe_seen(volatile struct mlx5_err_cqe *err_cqe) 59 { 60 static const uint8_t magic[] = "seen"; 61 int ret = 1; 62 unsigned int i; 63 64 for (i = 0; i < sizeof(magic); ++i) 65 if (!ret || err_cqe->rsvd1[i] != magic[i]) { 66 ret = 0; 67 err_cqe->rsvd1[i] = magic[i]; 68 } 69 return ret; 70 } 71 72 /** 73 * Handle error CQE. 74 * 75 * @param txq 76 * Pointer to TX queue structure. 77 * @param error_cqe 78 * Pointer to the error CQE. 79 * 80 * @return 81 * Negative value if queue recovery failed, otherwise 82 * the error completion entry is handled successfully. 83 */ 84 static int 85 mlx5_tx_error_cqe_handle(struct mlx5_txq_data *__rte_restrict txq, 86 volatile struct mlx5_err_cqe *err_cqe) 87 { 88 if (err_cqe->syndrome != MLX5_CQE_SYNDROME_WR_FLUSH_ERR) { 89 const uint16_t wqe_m = ((1 << txq->wqe_n) - 1); 90 struct mlx5_txq_ctrl *txq_ctrl = 91 container_of(txq, struct mlx5_txq_ctrl, txq); 92 uint16_t new_wqe_pi = rte_be_to_cpu_16(err_cqe->wqe_counter); 93 int seen = check_err_cqe_seen(err_cqe); 94 95 if (!seen && txq_ctrl->dump_file_n < 96 txq_ctrl->priv->config.max_dump_files_num) { 97 MKSTR(err_str, "Unexpected CQE error syndrome " 98 "0x%02x CQN = %u SQN = %u wqe_counter = %u " 99 "wq_ci = %u cq_ci = %u", err_cqe->syndrome, 100 txq->cqe_s, txq->qp_num_8s >> 8, 101 rte_be_to_cpu_16(err_cqe->wqe_counter), 102 txq->wqe_ci, txq->cq_ci); 103 MKSTR(name, "dpdk_mlx5_port_%u_txq_%u_index_%u_%u", 104 PORT_ID(txq_ctrl->priv), txq->idx, 105 txq_ctrl->dump_file_n, (uint32_t)rte_rdtsc()); 106 mlx5_dump_debug_information(name, NULL, err_str, 0); 107 mlx5_dump_debug_information(name, "MLX5 Error CQ:", 108 (const void *)((uintptr_t) 109 txq->cqes), 110 sizeof(*err_cqe) * 111 (1 << txq->cqe_n)); 112 mlx5_dump_debug_information(name, "MLX5 Error SQ:", 113 (const void *)((uintptr_t) 114 txq->wqes), 115 MLX5_WQE_SIZE * 116 (1 << txq->wqe_n)); 117 txq_ctrl->dump_file_n++; 118 } 119 if (!seen) 120 /* 121 * Count errors in WQEs units. 122 * Later it can be improved to count error packets, 123 * for example, by SQ parsing to find how much packets 124 * should be counted for each WQE. 125 */ 126 txq->stats.oerrors += ((txq->wqe_ci & wqe_m) - 127 new_wqe_pi) & wqe_m; 128 if (tx_recover_qp(txq_ctrl)) { 129 /* Recovering failed - retry later on the same WQE. */ 130 return -1; 131 } 132 /* Release all the remaining buffers. */ 133 txq_free_elts(txq_ctrl); 134 } 135 return 0; 136 } 137 138 /** 139 * Dummy DPDK callback for TX. 140 * 141 * This function is used to temporarily replace the real callback during 142 * unsafe control operations on the queue, or in case of error. 143 * 144 * @param dpdk_txq 145 * Generic pointer to TX queue structure. 146 * @param[in] pkts 147 * Packets to transmit. 148 * @param pkts_n 149 * Number of packets in array. 150 * 151 * @return 152 * Number of packets successfully transmitted (<= pkts_n). 153 */ 154 uint16_t 155 removed_tx_burst(void *dpdk_txq __rte_unused, 156 struct rte_mbuf **pkts __rte_unused, 157 uint16_t pkts_n __rte_unused) 158 { 159 rte_mb(); 160 return 0; 161 } 162 163 /** 164 * Update completion queue consuming index via doorbell 165 * and flush the completed data buffers. 166 * 167 * @param txq 168 * Pointer to TX queue structure. 169 * @param last_cqe 170 * valid CQE pointer, if not NULL update txq->wqe_pi and flush the buffers. 171 * @param olx 172 * Configured Tx offloads mask. It is fully defined at 173 * compile time and may be used for optimization. 174 */ 175 static __rte_always_inline void 176 mlx5_tx_comp_flush(struct mlx5_txq_data *__rte_restrict txq, 177 volatile struct mlx5_cqe *last_cqe, 178 unsigned int olx __rte_unused) 179 { 180 if (likely(last_cqe != NULL)) { 181 uint16_t tail; 182 183 txq->wqe_pi = rte_be_to_cpu_16(last_cqe->wqe_counter); 184 tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m]; 185 if (likely(tail != txq->elts_tail)) { 186 mlx5_tx_free_elts(txq, tail, olx); 187 MLX5_ASSERT(tail == txq->elts_tail); 188 } 189 } 190 } 191 192 /** 193 * Manage TX completions. This routine checks the CQ for 194 * arrived CQEs, deduces the last accomplished WQE in SQ, 195 * updates SQ producing index and frees all completed mbufs. 196 * 197 * @param txq 198 * Pointer to TX queue structure. 199 * @param olx 200 * Configured Tx offloads mask. It is fully defined at 201 * compile time and may be used for optimization. 202 * 203 * NOTE: not inlined intentionally, it makes tx_burst 204 * routine smaller, simple and faster - from experiments. 205 */ 206 void 207 mlx5_tx_handle_completion(struct mlx5_txq_data *__rte_restrict txq, 208 unsigned int olx __rte_unused) 209 { 210 unsigned int count = MLX5_TX_COMP_MAX_CQE; 211 volatile struct mlx5_cqe *last_cqe = NULL; 212 bool ring_doorbell = false; 213 int ret; 214 215 do { 216 volatile struct mlx5_cqe *cqe; 217 218 cqe = &txq->cqes[txq->cq_ci & txq->cqe_m]; 219 ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci); 220 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { 221 if (likely(ret != MLX5_CQE_STATUS_ERR)) { 222 /* No new CQEs in completion queue. */ 223 MLX5_ASSERT(ret == MLX5_CQE_STATUS_HW_OWN); 224 break; 225 } 226 /* 227 * Some error occurred, try to restart. 228 * We have no barrier after WQE related Doorbell 229 * written, make sure all writes are completed 230 * here, before we might perform SQ reset. 231 */ 232 rte_wmb(); 233 ret = mlx5_tx_error_cqe_handle 234 (txq, (volatile struct mlx5_err_cqe *)cqe); 235 if (unlikely(ret < 0)) { 236 /* 237 * Some error occurred on queue error 238 * handling, we do not advance the index 239 * here, allowing to retry on next call. 240 */ 241 return; 242 } 243 /* 244 * We are going to fetch all entries with 245 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status. 246 * The send queue is supposed to be empty. 247 */ 248 ring_doorbell = true; 249 ++txq->cq_ci; 250 txq->cq_pi = txq->cq_ci; 251 last_cqe = NULL; 252 continue; 253 } 254 /* Normal transmit completion. */ 255 MLX5_ASSERT(txq->cq_ci != txq->cq_pi); 256 #ifdef RTE_LIBRTE_MLX5_DEBUG 257 MLX5_ASSERT((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) == 258 cqe->wqe_counter); 259 #endif 260 ring_doorbell = true; 261 ++txq->cq_ci; 262 last_cqe = cqe; 263 /* 264 * We have to restrict the amount of processed CQEs 265 * in one tx_burst routine call. The CQ may be large 266 * and many CQEs may be updated by the NIC in one 267 * transaction. Buffers freeing is time consuming, 268 * multiple iterations may introduce significant latency. 269 */ 270 if (likely(--count == 0)) 271 break; 272 } while (true); 273 if (likely(ring_doorbell)) { 274 /* Ring doorbell to notify hardware. */ 275 rte_compiler_barrier(); 276 *txq->cq_db = rte_cpu_to_be_32(txq->cq_ci); 277 mlx5_tx_comp_flush(txq, last_cqe, olx); 278 } 279 } 280 281 /** 282 * DPDK callback to check the status of a Tx descriptor. 283 * 284 * @param tx_queue 285 * The Tx queue. 286 * @param[in] offset 287 * The index of the descriptor in the ring. 288 * 289 * @return 290 * The status of the Tx descriptor. 291 */ 292 int 293 mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset) 294 { 295 struct mlx5_txq_data *__rte_restrict txq = tx_queue; 296 uint16_t used; 297 298 mlx5_tx_handle_completion(txq, 0); 299 used = txq->elts_head - txq->elts_tail; 300 if (offset < used) 301 return RTE_ETH_TX_DESC_FULL; 302 return RTE_ETH_TX_DESC_DONE; 303 } 304 305 /* 306 * Array of declared and compiled Tx burst function and corresponding 307 * supported offloads set. The array is used to select the Tx burst 308 * function for specified offloads set at Tx queue configuration time. 309 */ 310 const struct { 311 eth_tx_burst_t func; 312 unsigned int olx; 313 } txoff_func[] = { 314 MLX5_TXOFF_INFO(full_empw, 315 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 316 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 317 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 318 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 319 320 MLX5_TXOFF_INFO(none_empw, 321 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW) 322 323 MLX5_TXOFF_INFO(md_empw, 324 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 325 326 MLX5_TXOFF_INFO(mt_empw, 327 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 328 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 329 330 MLX5_TXOFF_INFO(mtsc_empw, 331 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 332 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 333 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 334 335 MLX5_TXOFF_INFO(mti_empw, 336 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 337 MLX5_TXOFF_CONFIG_INLINE | 338 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 339 340 MLX5_TXOFF_INFO(mtv_empw, 341 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 342 MLX5_TXOFF_CONFIG_VLAN | 343 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 344 345 MLX5_TXOFF_INFO(mtiv_empw, 346 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 347 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 348 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 349 350 MLX5_TXOFF_INFO(sc_empw, 351 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 352 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 353 354 MLX5_TXOFF_INFO(sci_empw, 355 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 356 MLX5_TXOFF_CONFIG_INLINE | 357 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 358 359 MLX5_TXOFF_INFO(scv_empw, 360 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 361 MLX5_TXOFF_CONFIG_VLAN | 362 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 363 364 MLX5_TXOFF_INFO(sciv_empw, 365 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 366 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 367 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 368 369 MLX5_TXOFF_INFO(i_empw, 370 MLX5_TXOFF_CONFIG_INLINE | 371 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 372 373 MLX5_TXOFF_INFO(v_empw, 374 MLX5_TXOFF_CONFIG_VLAN | 375 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 376 377 MLX5_TXOFF_INFO(iv_empw, 378 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 379 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW) 380 381 MLX5_TXOFF_INFO(full_ts_nompw, 382 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP) 383 384 MLX5_TXOFF_INFO(full_ts_nompwi, 385 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 386 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 387 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA | 388 MLX5_TXOFF_CONFIG_TXPP) 389 390 MLX5_TXOFF_INFO(full_ts, 391 MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_TXPP | 392 MLX5_TXOFF_CONFIG_EMPW) 393 394 MLX5_TXOFF_INFO(full_ts_noi, 395 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 396 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 397 MLX5_TXOFF_CONFIG_VLAN | MLX5_TXOFF_CONFIG_METADATA | 398 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 399 400 MLX5_TXOFF_INFO(none_ts, 401 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_TXPP | 402 MLX5_TXOFF_CONFIG_EMPW) 403 404 MLX5_TXOFF_INFO(mdi_ts, 405 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA | 406 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 407 408 MLX5_TXOFF_INFO(mti_ts, 409 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 410 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_METADATA | 411 MLX5_TXOFF_CONFIG_TXPP | MLX5_TXOFF_CONFIG_EMPW) 412 413 MLX5_TXOFF_INFO(mtiv_ts, 414 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 415 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 416 MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_TXPP | 417 MLX5_TXOFF_CONFIG_EMPW) 418 419 MLX5_TXOFF_INFO(full, 420 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 421 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 422 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 423 MLX5_TXOFF_CONFIG_METADATA) 424 425 MLX5_TXOFF_INFO(none, 426 MLX5_TXOFF_CONFIG_NONE) 427 428 MLX5_TXOFF_INFO(md, 429 MLX5_TXOFF_CONFIG_METADATA) 430 431 MLX5_TXOFF_INFO(mt, 432 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 433 MLX5_TXOFF_CONFIG_METADATA) 434 435 MLX5_TXOFF_INFO(mtsc, 436 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 437 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 438 MLX5_TXOFF_CONFIG_METADATA) 439 440 MLX5_TXOFF_INFO(mti, 441 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 442 MLX5_TXOFF_CONFIG_INLINE | 443 MLX5_TXOFF_CONFIG_METADATA) 444 445 MLX5_TXOFF_INFO(mtv, 446 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 447 MLX5_TXOFF_CONFIG_VLAN | 448 MLX5_TXOFF_CONFIG_METADATA) 449 450 MLX5_TXOFF_INFO(mtiv, 451 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO | 452 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 453 MLX5_TXOFF_CONFIG_METADATA) 454 455 MLX5_TXOFF_INFO(sc, 456 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 457 MLX5_TXOFF_CONFIG_METADATA) 458 459 MLX5_TXOFF_INFO(sci, 460 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 461 MLX5_TXOFF_CONFIG_INLINE | 462 MLX5_TXOFF_CONFIG_METADATA) 463 464 MLX5_TXOFF_INFO(scv, 465 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 466 MLX5_TXOFF_CONFIG_VLAN | 467 MLX5_TXOFF_CONFIG_METADATA) 468 469 MLX5_TXOFF_INFO(sciv, 470 MLX5_TXOFF_CONFIG_SWP | MLX5_TXOFF_CONFIG_CSUM | 471 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 472 MLX5_TXOFF_CONFIG_METADATA) 473 474 MLX5_TXOFF_INFO(i, 475 MLX5_TXOFF_CONFIG_INLINE | 476 MLX5_TXOFF_CONFIG_METADATA) 477 478 MLX5_TXOFF_INFO(v, 479 MLX5_TXOFF_CONFIG_VLAN | 480 MLX5_TXOFF_CONFIG_METADATA) 481 482 MLX5_TXOFF_INFO(iv, 483 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN | 484 MLX5_TXOFF_CONFIG_METADATA) 485 486 MLX5_TXOFF_INFO(none_mpw, 487 MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW | 488 MLX5_TXOFF_CONFIG_MPW) 489 490 MLX5_TXOFF_INFO(mci_mpw, 491 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 492 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 493 MLX5_TXOFF_CONFIG_MPW) 494 495 MLX5_TXOFF_INFO(mc_mpw, 496 MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_CSUM | 497 MLX5_TXOFF_CONFIG_EMPW | MLX5_TXOFF_CONFIG_MPW) 498 499 MLX5_TXOFF_INFO(i_mpw, 500 MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_EMPW | 501 MLX5_TXOFF_CONFIG_MPW) 502 }; 503 504 /** 505 * Configure the Tx function to use. The routine checks configured 506 * Tx offloads for the device and selects appropriate Tx burst routine. 507 * There are multiple Tx burst routines compiled from the same template 508 * in the most optimal way for the dedicated Tx offloads set. 509 * 510 * @param dev 511 * Pointer to private data structure. 512 * 513 * @return 514 * Pointer to selected Tx burst function. 515 */ 516 eth_tx_burst_t 517 mlx5_select_tx_function(struct rte_eth_dev *dev) 518 { 519 struct mlx5_priv *priv = dev->data->dev_private; 520 struct mlx5_dev_config *config = &priv->config; 521 uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; 522 unsigned int diff = 0, olx = 0, i, m; 523 524 MLX5_ASSERT(priv); 525 if (tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) { 526 /* We should support Multi-Segment Packets. */ 527 olx |= MLX5_TXOFF_CONFIG_MULTI; 528 } 529 if (tx_offloads & (RTE_ETH_TX_OFFLOAD_TCP_TSO | 530 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 531 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | 532 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | 533 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO)) { 534 /* We should support TCP Send Offload. */ 535 olx |= MLX5_TXOFF_CONFIG_TSO; 536 } 537 if (tx_offloads & (RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | 538 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO | 539 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { 540 /* We should support Software Parser for Tunnels. */ 541 olx |= MLX5_TXOFF_CONFIG_SWP; 542 } 543 if (tx_offloads & (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 544 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 545 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 546 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { 547 /* We should support IP/TCP/UDP Checksums. */ 548 olx |= MLX5_TXOFF_CONFIG_CSUM; 549 } 550 if (tx_offloads & RTE_ETH_TX_OFFLOAD_VLAN_INSERT) { 551 /* We should support VLAN insertion. */ 552 olx |= MLX5_TXOFF_CONFIG_VLAN; 553 } 554 if (tx_offloads & RTE_ETH_TX_OFFLOAD_SEND_ON_TIMESTAMP && 555 rte_mbuf_dynflag_lookup 556 (RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 && 557 rte_mbuf_dynfield_lookup 558 (RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) { 559 /* Offload configured, dynamic entities registered. */ 560 olx |= MLX5_TXOFF_CONFIG_TXPP; 561 } 562 if (priv->txqs_n && (*priv->txqs)[0]) { 563 struct mlx5_txq_data *txd = (*priv->txqs)[0]; 564 565 if (txd->inlen_send) { 566 /* 567 * Check the data inline requirements. Data inline 568 * is enabled on per device basis, we can check 569 * the first Tx queue only. 570 * 571 * If device does not support VLAN insertion in WQE 572 * and some queues are requested to perform VLAN 573 * insertion offload than inline must be enabled. 574 */ 575 olx |= MLX5_TXOFF_CONFIG_INLINE; 576 } 577 } 578 if (config->mps == MLX5_MPW_ENHANCED && 579 config->txq_inline_min <= 0) { 580 /* 581 * The NIC supports Enhanced Multi-Packet Write 582 * and does not require minimal inline data. 583 */ 584 olx |= MLX5_TXOFF_CONFIG_EMPW; 585 } 586 if (rte_flow_dynf_metadata_avail()) { 587 /* We should support Flow metadata. */ 588 olx |= MLX5_TXOFF_CONFIG_METADATA; 589 } 590 if (config->mps == MLX5_MPW) { 591 /* 592 * The NIC supports Legacy Multi-Packet Write. 593 * The MLX5_TXOFF_CONFIG_MPW controls the descriptor building 594 * method in combination with MLX5_TXOFF_CONFIG_EMPW. 595 */ 596 if (!(olx & (MLX5_TXOFF_CONFIG_TSO | 597 MLX5_TXOFF_CONFIG_SWP | 598 MLX5_TXOFF_CONFIG_VLAN | 599 MLX5_TXOFF_CONFIG_METADATA))) 600 olx |= MLX5_TXOFF_CONFIG_EMPW | 601 MLX5_TXOFF_CONFIG_MPW; 602 } 603 /* 604 * Scan the routines table to find the minimal 605 * satisfying routine with requested offloads. 606 */ 607 m = RTE_DIM(txoff_func); 608 for (i = 0; i < RTE_DIM(txoff_func); i++) { 609 unsigned int tmp; 610 611 tmp = txoff_func[i].olx; 612 if (tmp == olx) { 613 /* Meets requested offloads exactly.*/ 614 m = i; 615 break; 616 } 617 if ((tmp & olx) != olx) { 618 /* Does not meet requested offloads at all. */ 619 continue; 620 } 621 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW) 622 /* Do not enable legacy MPW if not configured. */ 623 continue; 624 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW) 625 /* Do not enable eMPW if not configured. */ 626 continue; 627 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE) 628 /* Do not enable inlining if not configured. */ 629 continue; 630 if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP) 631 /* Do not enable scheduling if not configured. */ 632 continue; 633 /* 634 * Some routine meets the requirements. 635 * Check whether it has minimal amount 636 * of not requested offloads. 637 */ 638 tmp = __builtin_popcountl(tmp & ~olx); 639 if (m >= RTE_DIM(txoff_func) || tmp < diff) { 640 /* First or better match, save and continue. */ 641 m = i; 642 diff = tmp; 643 continue; 644 } 645 if (tmp == diff) { 646 tmp = txoff_func[i].olx ^ txoff_func[m].olx; 647 if (__builtin_ffsl(txoff_func[i].olx & ~tmp) < 648 __builtin_ffsl(txoff_func[m].olx & ~tmp)) { 649 /* Lighter not requested offload. */ 650 m = i; 651 } 652 } 653 } 654 if (m >= RTE_DIM(txoff_func)) { 655 DRV_LOG(DEBUG, "port %u has no selected Tx function" 656 " for requested offloads %04X", 657 dev->data->port_id, olx); 658 return NULL; 659 } 660 DRV_LOG(DEBUG, "port %u has selected Tx function" 661 " supporting offloads %04X/%04X", 662 dev->data->port_id, olx, txoff_func[m].olx); 663 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI) 664 DRV_LOG(DEBUG, "\tMULTI (multi segment)"); 665 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO) 666 DRV_LOG(DEBUG, "\tTSO (TCP send offload)"); 667 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP) 668 DRV_LOG(DEBUG, "\tSWP (software parser)"); 669 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM) 670 DRV_LOG(DEBUG, "\tCSUM (checksum offload)"); 671 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE) 672 DRV_LOG(DEBUG, "\tINLIN (inline data)"); 673 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN) 674 DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)"); 675 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA) 676 DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)"); 677 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP) 678 DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)"); 679 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) { 680 if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW) 681 DRV_LOG(DEBUG, "\tMPW (Legacy MPW)"); 682 else 683 DRV_LOG(DEBUG, "\tEMPW (Enhanced MPW)"); 684 } 685 return txoff_func[m].func; 686 } 687 688 /** 689 * DPDK callback to get the TX queue information. 690 * 691 * @param dev 692 * Pointer to the device structure. 693 * 694 * @param tx_queue_id 695 * Tx queue identificator. 696 * 697 * @param qinfo 698 * Pointer to the TX queue information structure. 699 * 700 * @return 701 * None. 702 */ 703 void 704 mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id, 705 struct rte_eth_txq_info *qinfo) 706 { 707 struct mlx5_priv *priv = dev->data->dev_private; 708 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id]; 709 struct mlx5_txq_ctrl *txq_ctrl = 710 container_of(txq, struct mlx5_txq_ctrl, txq); 711 712 if (!txq) 713 return; 714 qinfo->nb_desc = txq->elts_s; 715 qinfo->conf.tx_thresh.pthresh = 0; 716 qinfo->conf.tx_thresh.hthresh = 0; 717 qinfo->conf.tx_thresh.wthresh = 0; 718 qinfo->conf.tx_rs_thresh = 0; 719 qinfo->conf.tx_free_thresh = 0; 720 qinfo->conf.tx_deferred_start = txq_ctrl ? 0 : 1; 721 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; 722 } 723 724 /** 725 * DPDK callback to get the TX packet burst mode information. 726 * 727 * @param dev 728 * Pointer to the device structure. 729 * 730 * @param tx_queue_id 731 * Tx queue identificatior. 732 * 733 * @param mode 734 * Pointer to the burts mode information. 735 * 736 * @return 737 * 0 as success, -EINVAL as failure. 738 */ 739 int 740 mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, 741 uint16_t tx_queue_id, 742 struct rte_eth_burst_mode *mode) 743 { 744 eth_tx_burst_t pkt_burst = dev->tx_pkt_burst; 745 struct mlx5_priv *priv = dev->data->dev_private; 746 struct mlx5_txq_data *txq = (*priv->txqs)[tx_queue_id]; 747 unsigned int i, olx; 748 749 for (i = 0; i < RTE_DIM(txoff_func); i++) { 750 if (pkt_burst == txoff_func[i].func) { 751 olx = txoff_func[i].olx; 752 snprintf(mode->info, sizeof(mode->info), 753 "%s%s%s%s%s%s%s%s%s%s", 754 (olx & MLX5_TXOFF_CONFIG_EMPW) ? 755 ((olx & MLX5_TXOFF_CONFIG_MPW) ? 756 "Legacy MPW" : "Enhanced MPW") : "No MPW", 757 (olx & MLX5_TXOFF_CONFIG_MULTI) ? 758 " + MULTI" : "", 759 (olx & MLX5_TXOFF_CONFIG_TSO) ? 760 " + TSO" : "", 761 (olx & MLX5_TXOFF_CONFIG_SWP) ? 762 " + SWP" : "", 763 (olx & MLX5_TXOFF_CONFIG_CSUM) ? 764 " + CSUM" : "", 765 (olx & MLX5_TXOFF_CONFIG_INLINE) ? 766 " + INLINE" : "", 767 (olx & MLX5_TXOFF_CONFIG_VLAN) ? 768 " + VLAN" : "", 769 (olx & MLX5_TXOFF_CONFIG_METADATA) ? 770 " + METADATA" : "", 771 (olx & MLX5_TXOFF_CONFIG_TXPP) ? 772 " + TXPP" : "", 773 (txq && txq->fast_free) ? 774 " + Fast Free" : ""); 775 return 0; 776 } 777 } 778 return -EINVAL; 779 } 780