1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <unistd.h> 7 8 #include <rte_ether.h> 9 #include <ethdev_driver.h> 10 #include <rte_interrupts.h> 11 #include <rte_alarm.h> 12 #include <rte_cycles.h> 13 14 #include <mlx5_malloc.h> 15 16 #include "mlx5.h" 17 #include "mlx5_flow.h" 18 #include "mlx5_rx.h" 19 #include "mlx5_tx.h" 20 #include "mlx5_utils.h" 21 #include "rte_pmd_mlx5.h" 22 23 /** 24 * Stop traffic on Tx queues. 25 * 26 * @param dev 27 * Pointer to Ethernet device structure. 28 */ 29 static void 30 mlx5_txq_stop(struct rte_eth_dev *dev) 31 { 32 struct mlx5_priv *priv = dev->data->dev_private; 33 unsigned int i; 34 35 for (i = 0; i != priv->txqs_n; ++i) 36 mlx5_txq_release(dev, i); 37 } 38 39 /** 40 * Start traffic on Tx queues. 41 * 42 * @param dev 43 * Pointer to Ethernet device structure. 44 * 45 * @return 46 * 0 on success, a negative errno value otherwise and rte_errno is set. 47 */ 48 static int 49 mlx5_txq_start(struct rte_eth_dev *dev) 50 { 51 struct mlx5_priv *priv = dev->data->dev_private; 52 unsigned int i; 53 int ret; 54 55 for (i = 0; i != priv->txqs_n; ++i) { 56 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 57 struct mlx5_txq_data *txq_data = &txq_ctrl->txq; 58 uint32_t flags = MLX5_MEM_RTE | MLX5_MEM_ZERO; 59 60 if (!txq_ctrl) 61 continue; 62 if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD) 63 txq_alloc_elts(txq_ctrl); 64 MLX5_ASSERT(!txq_ctrl->obj); 65 txq_ctrl->obj = mlx5_malloc(flags, sizeof(struct mlx5_txq_obj), 66 0, txq_ctrl->socket); 67 if (!txq_ctrl->obj) { 68 DRV_LOG(ERR, "Port %u Tx queue %u cannot allocate " 69 "memory resources.", dev->data->port_id, 70 txq_data->idx); 71 rte_errno = ENOMEM; 72 goto error; 73 } 74 ret = priv->obj_ops.txq_obj_new(dev, i); 75 if (ret < 0) { 76 mlx5_free(txq_ctrl->obj); 77 txq_ctrl->obj = NULL; 78 goto error; 79 } 80 if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD) { 81 size_t size = txq_data->cqe_s * sizeof(*txq_data->fcqs); 82 83 txq_data->fcqs = mlx5_malloc(flags, size, 84 RTE_CACHE_LINE_SIZE, 85 txq_ctrl->socket); 86 if (!txq_data->fcqs) { 87 DRV_LOG(ERR, "Port %u Tx queue %u cannot " 88 "allocate memory (FCQ).", 89 dev->data->port_id, i); 90 rte_errno = ENOMEM; 91 goto error; 92 } 93 } 94 DRV_LOG(DEBUG, "Port %u txq %u updated with %p.", 95 dev->data->port_id, i, (void *)&txq_ctrl->obj); 96 LIST_INSERT_HEAD(&priv->txqsobj, txq_ctrl->obj, next); 97 } 98 return 0; 99 error: 100 ret = rte_errno; /* Save rte_errno before cleanup. */ 101 do { 102 mlx5_txq_release(dev, i); 103 } while (i-- != 0); 104 rte_errno = ret; /* Restore rte_errno. */ 105 return -rte_errno; 106 } 107 108 /** 109 * Translate the chunk address to MR key in order to put in into the cache. 110 */ 111 static void 112 mlx5_rxq_mempool_register_cb(struct rte_mempool *mp, void *opaque, 113 struct rte_mempool_memhdr *memhdr, 114 unsigned int idx) 115 { 116 struct mlx5_rxq_data *rxq = opaque; 117 118 RTE_SET_USED(mp); 119 RTE_SET_USED(idx); 120 mlx5_rx_addr2mr(rxq, (uintptr_t)memhdr->addr); 121 } 122 123 /** 124 * Register Rx queue mempools and fill the Rx queue cache. 125 * This function tolerates repeated mempool registration. 126 * 127 * @param[in] rxq_ctrl 128 * Rx queue control data. 129 * 130 * @return 131 * 0 on success, (-1) on failure and rte_errno is set. 132 */ 133 static int 134 mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl) 135 { 136 struct rte_mempool *mp; 137 uint32_t s; 138 int ret = 0; 139 140 mlx5_mr_flush_local_cache(&rxq_ctrl->rxq.mr_ctrl); 141 /* MPRQ mempool is registered on creation, just fill the cache. */ 142 if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq)) { 143 rte_mempool_mem_iter(rxq_ctrl->rxq.mprq_mp, 144 mlx5_rxq_mempool_register_cb, 145 &rxq_ctrl->rxq); 146 return 0; 147 } 148 for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) { 149 uint32_t flags; 150 151 mp = rxq_ctrl->rxq.rxseg[s].mp; 152 flags = rte_pktmbuf_priv_flags(mp); 153 ret = mlx5_mr_mempool_register(rxq_ctrl->sh->cdev, mp); 154 if (ret < 0 && rte_errno != EEXIST) 155 return ret; 156 if ((flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) == 0) 157 rte_mempool_mem_iter(mp, mlx5_rxq_mempool_register_cb, 158 &rxq_ctrl->rxq); 159 } 160 return 0; 161 } 162 163 /** 164 * Stop traffic on Rx queues. 165 * 166 * @param dev 167 * Pointer to Ethernet device structure. 168 */ 169 static void 170 mlx5_rxq_stop(struct rte_eth_dev *dev) 171 { 172 struct mlx5_priv *priv = dev->data->dev_private; 173 unsigned int i; 174 175 for (i = 0; i != priv->rxqs_n; ++i) 176 mlx5_rxq_release(dev, i); 177 } 178 179 static int 180 mlx5_rxq_ctrl_prepare(struct rte_eth_dev *dev, struct mlx5_rxq_ctrl *rxq_ctrl, 181 unsigned int idx) 182 { 183 int ret = 0; 184 185 if (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD) { 186 /* 187 * Pre-register the mempools. Regardless of whether 188 * the implicit registration is enabled or not, 189 * Rx mempool destruction is tracked to free MRs. 190 */ 191 if (mlx5_rxq_mempool_register(rxq_ctrl) < 0) 192 return -rte_errno; 193 ret = rxq_alloc_elts(rxq_ctrl); 194 if (ret) 195 return ret; 196 } 197 MLX5_ASSERT(!rxq_ctrl->obj); 198 rxq_ctrl->obj = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 199 sizeof(*rxq_ctrl->obj), 0, 200 rxq_ctrl->socket); 201 if (!rxq_ctrl->obj) { 202 DRV_LOG(ERR, "Port %u Rx queue %u can't allocate resources.", 203 dev->data->port_id, idx); 204 rte_errno = ENOMEM; 205 return -rte_errno; 206 } 207 DRV_LOG(DEBUG, "Port %u rxq %u updated with %p.", dev->data->port_id, 208 idx, (void *)&rxq_ctrl->obj); 209 return 0; 210 } 211 212 /** 213 * Start traffic on Rx queues. 214 * 215 * @param dev 216 * Pointer to Ethernet device structure. 217 * 218 * @return 219 * 0 on success, a negative errno value otherwise and rte_errno is set. 220 */ 221 static int 222 mlx5_rxq_start(struct rte_eth_dev *dev) 223 { 224 struct mlx5_priv *priv = dev->data->dev_private; 225 unsigned int i; 226 int ret = 0; 227 228 /* Allocate/reuse/resize mempool for Multi-Packet RQ. */ 229 if (mlx5_mprq_alloc_mp(dev)) { 230 /* Should not release Rx queues but return immediately. */ 231 return -rte_errno; 232 } 233 DRV_LOG(DEBUG, "Port %u device_attr.max_qp_wr is %d.", 234 dev->data->port_id, priv->sh->device_attr.max_qp_wr); 235 DRV_LOG(DEBUG, "Port %u device_attr.max_sge is %d.", 236 dev->data->port_id, priv->sh->device_attr.max_sge); 237 for (i = 0; i != priv->rxqs_n; ++i) { 238 struct mlx5_rxq_priv *rxq = mlx5_rxq_ref(dev, i); 239 struct mlx5_rxq_ctrl *rxq_ctrl; 240 241 if (rxq == NULL) 242 continue; 243 rxq_ctrl = rxq->ctrl; 244 if (!rxq_ctrl->started) { 245 if (mlx5_rxq_ctrl_prepare(dev, rxq_ctrl, i) < 0) 246 goto error; 247 LIST_INSERT_HEAD(&priv->rxqsobj, rxq_ctrl->obj, next); 248 } 249 ret = priv->obj_ops.rxq_obj_new(rxq); 250 if (ret) { 251 mlx5_free(rxq_ctrl->obj); 252 rxq_ctrl->obj = NULL; 253 goto error; 254 } 255 rxq_ctrl->started = true; 256 } 257 return 0; 258 error: 259 ret = rte_errno; /* Save rte_errno before cleanup. */ 260 do { 261 mlx5_rxq_release(dev, i); 262 } while (i-- != 0); 263 rte_errno = ret; /* Restore rte_errno. */ 264 return -rte_errno; 265 } 266 267 /** 268 * Binds Tx queues to Rx queues for hairpin. 269 * 270 * Binds Tx queues to the target Rx queues. 271 * 272 * @param dev 273 * Pointer to Ethernet device structure. 274 * 275 * @return 276 * 0 on success, a negative errno value otherwise and rte_errno is set. 277 */ 278 static int 279 mlx5_hairpin_auto_bind(struct rte_eth_dev *dev) 280 { 281 struct mlx5_priv *priv = dev->data->dev_private; 282 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 283 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 284 struct mlx5_txq_ctrl *txq_ctrl; 285 struct mlx5_rxq_priv *rxq; 286 struct mlx5_rxq_ctrl *rxq_ctrl; 287 struct mlx5_devx_obj *sq; 288 struct mlx5_devx_obj *rq; 289 unsigned int i; 290 int ret = 0; 291 bool need_auto = false; 292 uint16_t self_port = dev->data->port_id; 293 294 for (i = 0; i != priv->txqs_n; ++i) { 295 txq_ctrl = mlx5_txq_get(dev, i); 296 if (!txq_ctrl) 297 continue; 298 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN || 299 txq_ctrl->hairpin_conf.peers[0].port != self_port) { 300 mlx5_txq_release(dev, i); 301 continue; 302 } 303 if (txq_ctrl->hairpin_conf.manual_bind) { 304 mlx5_txq_release(dev, i); 305 return 0; 306 } 307 need_auto = true; 308 mlx5_txq_release(dev, i); 309 } 310 if (!need_auto) 311 return 0; 312 for (i = 0; i != priv->txqs_n; ++i) { 313 txq_ctrl = mlx5_txq_get(dev, i); 314 if (!txq_ctrl) 315 continue; 316 /* Skip hairpin queues with other peer ports. */ 317 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN || 318 txq_ctrl->hairpin_conf.peers[0].port != self_port) { 319 mlx5_txq_release(dev, i); 320 continue; 321 } 322 if (!txq_ctrl->obj) { 323 rte_errno = ENOMEM; 324 DRV_LOG(ERR, "port %u no txq object found: %d", 325 dev->data->port_id, i); 326 mlx5_txq_release(dev, i); 327 return -rte_errno; 328 } 329 sq = txq_ctrl->obj->sq; 330 rxq = mlx5_rxq_get(dev, txq_ctrl->hairpin_conf.peers[0].queue); 331 if (rxq == NULL) { 332 mlx5_txq_release(dev, i); 333 rte_errno = EINVAL; 334 DRV_LOG(ERR, "port %u no rxq object found: %d", 335 dev->data->port_id, 336 txq_ctrl->hairpin_conf.peers[0].queue); 337 return -rte_errno; 338 } 339 rxq_ctrl = rxq->ctrl; 340 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN || 341 rxq->hairpin_conf.peers[0].queue != i) { 342 rte_errno = ENOMEM; 343 DRV_LOG(ERR, "port %u Tx queue %d can't be binded to " 344 "Rx queue %d", dev->data->port_id, 345 i, txq_ctrl->hairpin_conf.peers[0].queue); 346 goto error; 347 } 348 rq = rxq_ctrl->obj->rq; 349 if (!rq) { 350 rte_errno = ENOMEM; 351 DRV_LOG(ERR, "port %u hairpin no matching rxq: %d", 352 dev->data->port_id, 353 txq_ctrl->hairpin_conf.peers[0].queue); 354 goto error; 355 } 356 sq_attr.state = MLX5_SQC_STATE_RDY; 357 sq_attr.sq_state = MLX5_SQC_STATE_RST; 358 sq_attr.hairpin_peer_rq = rq->id; 359 sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 360 ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr); 361 if (ret) 362 goto error; 363 rq_attr.state = MLX5_SQC_STATE_RDY; 364 rq_attr.rq_state = MLX5_SQC_STATE_RST; 365 rq_attr.hairpin_peer_sq = sq->id; 366 rq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 367 ret = mlx5_devx_cmd_modify_rq(rq, &rq_attr); 368 if (ret) 369 goto error; 370 /* Qs with auto-bind will be destroyed directly. */ 371 rxq->hairpin_status = 1; 372 txq_ctrl->hairpin_status = 1; 373 mlx5_txq_release(dev, i); 374 } 375 return 0; 376 error: 377 mlx5_txq_release(dev, i); 378 return -rte_errno; 379 } 380 381 /* 382 * Fetch the peer queue's SW & HW information. 383 * 384 * @param dev 385 * Pointer to Ethernet device structure. 386 * @param peer_queue 387 * Index of the queue to fetch the information. 388 * @param current_info 389 * Pointer to the input peer information, not used currently. 390 * @param peer_info 391 * Pointer to the structure to store the information, output. 392 * @param direction 393 * Positive to get the RxQ information, zero to get the TxQ information. 394 * 395 * @return 396 * 0 on success, a negative errno value otherwise and rte_errno is set. 397 */ 398 int 399 mlx5_hairpin_queue_peer_update(struct rte_eth_dev *dev, uint16_t peer_queue, 400 struct rte_hairpin_peer_info *current_info, 401 struct rte_hairpin_peer_info *peer_info, 402 uint32_t direction) 403 { 404 struct mlx5_priv *priv = dev->data->dev_private; 405 RTE_SET_USED(current_info); 406 407 if (dev->data->dev_started == 0) { 408 rte_errno = EBUSY; 409 DRV_LOG(ERR, "peer port %u is not started", 410 dev->data->port_id); 411 return -rte_errno; 412 } 413 /* 414 * Peer port used as egress. In the current design, hairpin Tx queue 415 * will be bound to the peer Rx queue. Indeed, only the information of 416 * peer Rx queue needs to be fetched. 417 */ 418 if (direction == 0) { 419 struct mlx5_txq_ctrl *txq_ctrl; 420 421 txq_ctrl = mlx5_txq_get(dev, peer_queue); 422 if (txq_ctrl == NULL) { 423 rte_errno = EINVAL; 424 DRV_LOG(ERR, "Failed to get port %u Tx queue %d", 425 dev->data->port_id, peer_queue); 426 return -rte_errno; 427 } 428 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 429 rte_errno = EINVAL; 430 DRV_LOG(ERR, "port %u queue %d is not a hairpin Txq", 431 dev->data->port_id, peer_queue); 432 mlx5_txq_release(dev, peer_queue); 433 return -rte_errno; 434 } 435 if (txq_ctrl->obj == NULL || txq_ctrl->obj->sq == NULL) { 436 rte_errno = ENOMEM; 437 DRV_LOG(ERR, "port %u no Txq object found: %d", 438 dev->data->port_id, peer_queue); 439 mlx5_txq_release(dev, peer_queue); 440 return -rte_errno; 441 } 442 peer_info->qp_id = txq_ctrl->obj->sq->id; 443 peer_info->vhca_id = priv->config.hca_attr.vhca_id; 444 /* 1-to-1 mapping, only the first one is used. */ 445 peer_info->peer_q = txq_ctrl->hairpin_conf.peers[0].queue; 446 peer_info->tx_explicit = txq_ctrl->hairpin_conf.tx_explicit; 447 peer_info->manual_bind = txq_ctrl->hairpin_conf.manual_bind; 448 mlx5_txq_release(dev, peer_queue); 449 } else { /* Peer port used as ingress. */ 450 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, peer_queue); 451 struct mlx5_rxq_ctrl *rxq_ctrl; 452 453 if (rxq == NULL) { 454 rte_errno = EINVAL; 455 DRV_LOG(ERR, "Failed to get port %u Rx queue %d", 456 dev->data->port_id, peer_queue); 457 return -rte_errno; 458 } 459 rxq_ctrl = rxq->ctrl; 460 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) { 461 rte_errno = EINVAL; 462 DRV_LOG(ERR, "port %u queue %d is not a hairpin Rxq", 463 dev->data->port_id, peer_queue); 464 return -rte_errno; 465 } 466 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) { 467 rte_errno = ENOMEM; 468 DRV_LOG(ERR, "port %u no Rxq object found: %d", 469 dev->data->port_id, peer_queue); 470 return -rte_errno; 471 } 472 peer_info->qp_id = rxq_ctrl->obj->rq->id; 473 peer_info->vhca_id = priv->config.hca_attr.vhca_id; 474 peer_info->peer_q = rxq->hairpin_conf.peers[0].queue; 475 peer_info->tx_explicit = rxq->hairpin_conf.tx_explicit; 476 peer_info->manual_bind = rxq->hairpin_conf.manual_bind; 477 } 478 return 0; 479 } 480 481 /* 482 * Bind the hairpin queue with the peer HW information. 483 * This needs to be called twice both for Tx and Rx queues of a pair. 484 * If the queue is already bound, it is considered successful. 485 * 486 * @param dev 487 * Pointer to Ethernet device structure. 488 * @param cur_queue 489 * Index of the queue to change the HW configuration to bind. 490 * @param peer_info 491 * Pointer to information of the peer queue. 492 * @param direction 493 * Positive to configure the TxQ, zero to configure the RxQ. 494 * 495 * @return 496 * 0 on success, a negative errno value otherwise and rte_errno is set. 497 */ 498 int 499 mlx5_hairpin_queue_peer_bind(struct rte_eth_dev *dev, uint16_t cur_queue, 500 struct rte_hairpin_peer_info *peer_info, 501 uint32_t direction) 502 { 503 int ret = 0; 504 505 /* 506 * Consistency checking of the peer queue: opposite direction is used 507 * to get the peer queue info with ethdev port ID, no need to check. 508 */ 509 if (peer_info->peer_q != cur_queue) { 510 rte_errno = EINVAL; 511 DRV_LOG(ERR, "port %u queue %d and peer queue %d mismatch", 512 dev->data->port_id, cur_queue, peer_info->peer_q); 513 return -rte_errno; 514 } 515 if (direction != 0) { 516 struct mlx5_txq_ctrl *txq_ctrl; 517 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 518 519 txq_ctrl = mlx5_txq_get(dev, cur_queue); 520 if (txq_ctrl == NULL) { 521 rte_errno = EINVAL; 522 DRV_LOG(ERR, "Failed to get port %u Tx queue %d", 523 dev->data->port_id, cur_queue); 524 return -rte_errno; 525 } 526 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 527 rte_errno = EINVAL; 528 DRV_LOG(ERR, "port %u queue %d not a hairpin Txq", 529 dev->data->port_id, cur_queue); 530 mlx5_txq_release(dev, cur_queue); 531 return -rte_errno; 532 } 533 if (txq_ctrl->obj == NULL || txq_ctrl->obj->sq == NULL) { 534 rte_errno = ENOMEM; 535 DRV_LOG(ERR, "port %u no Txq object found: %d", 536 dev->data->port_id, cur_queue); 537 mlx5_txq_release(dev, cur_queue); 538 return -rte_errno; 539 } 540 if (txq_ctrl->hairpin_status != 0) { 541 DRV_LOG(DEBUG, "port %u Tx queue %d is already bound", 542 dev->data->port_id, cur_queue); 543 mlx5_txq_release(dev, cur_queue); 544 return 0; 545 } 546 /* 547 * All queues' of one port consistency checking is done in the 548 * bind() function, and that is optional. 549 */ 550 if (peer_info->tx_explicit != 551 txq_ctrl->hairpin_conf.tx_explicit) { 552 rte_errno = EINVAL; 553 DRV_LOG(ERR, "port %u Tx queue %d and peer Tx rule mode" 554 " mismatch", dev->data->port_id, cur_queue); 555 mlx5_txq_release(dev, cur_queue); 556 return -rte_errno; 557 } 558 if (peer_info->manual_bind != 559 txq_ctrl->hairpin_conf.manual_bind) { 560 rte_errno = EINVAL; 561 DRV_LOG(ERR, "port %u Tx queue %d and peer binding mode" 562 " mismatch", dev->data->port_id, cur_queue); 563 mlx5_txq_release(dev, cur_queue); 564 return -rte_errno; 565 } 566 sq_attr.state = MLX5_SQC_STATE_RDY; 567 sq_attr.sq_state = MLX5_SQC_STATE_RST; 568 sq_attr.hairpin_peer_rq = peer_info->qp_id; 569 sq_attr.hairpin_peer_vhca = peer_info->vhca_id; 570 ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq, &sq_attr); 571 if (ret == 0) 572 txq_ctrl->hairpin_status = 1; 573 mlx5_txq_release(dev, cur_queue); 574 } else { 575 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, cur_queue); 576 struct mlx5_rxq_ctrl *rxq_ctrl; 577 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 578 579 if (rxq == NULL) { 580 rte_errno = EINVAL; 581 DRV_LOG(ERR, "Failed to get port %u Rx queue %d", 582 dev->data->port_id, cur_queue); 583 return -rte_errno; 584 } 585 rxq_ctrl = rxq->ctrl; 586 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) { 587 rte_errno = EINVAL; 588 DRV_LOG(ERR, "port %u queue %d not a hairpin Rxq", 589 dev->data->port_id, cur_queue); 590 return -rte_errno; 591 } 592 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) { 593 rte_errno = ENOMEM; 594 DRV_LOG(ERR, "port %u no Rxq object found: %d", 595 dev->data->port_id, cur_queue); 596 return -rte_errno; 597 } 598 if (rxq->hairpin_status != 0) { 599 DRV_LOG(DEBUG, "port %u Rx queue %d is already bound", 600 dev->data->port_id, cur_queue); 601 return 0; 602 } 603 if (peer_info->tx_explicit != 604 rxq->hairpin_conf.tx_explicit) { 605 rte_errno = EINVAL; 606 DRV_LOG(ERR, "port %u Rx queue %d and peer Tx rule mode" 607 " mismatch", dev->data->port_id, cur_queue); 608 return -rte_errno; 609 } 610 if (peer_info->manual_bind != 611 rxq->hairpin_conf.manual_bind) { 612 rte_errno = EINVAL; 613 DRV_LOG(ERR, "port %u Rx queue %d and peer binding mode" 614 " mismatch", dev->data->port_id, cur_queue); 615 return -rte_errno; 616 } 617 rq_attr.state = MLX5_SQC_STATE_RDY; 618 rq_attr.rq_state = MLX5_SQC_STATE_RST; 619 rq_attr.hairpin_peer_sq = peer_info->qp_id; 620 rq_attr.hairpin_peer_vhca = peer_info->vhca_id; 621 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr); 622 if (ret == 0) 623 rxq->hairpin_status = 1; 624 } 625 return ret; 626 } 627 628 /* 629 * Unbind the hairpin queue and reset its HW configuration. 630 * This needs to be called twice both for Tx and Rx queues of a pair. 631 * If the queue is already unbound, it is considered successful. 632 * 633 * @param dev 634 * Pointer to Ethernet device structure. 635 * @param cur_queue 636 * Index of the queue to change the HW configuration to unbind. 637 * @param direction 638 * Positive to reset the TxQ, zero to reset the RxQ. 639 * 640 * @return 641 * 0 on success, a negative errno value otherwise and rte_errno is set. 642 */ 643 int 644 mlx5_hairpin_queue_peer_unbind(struct rte_eth_dev *dev, uint16_t cur_queue, 645 uint32_t direction) 646 { 647 int ret = 0; 648 649 if (direction != 0) { 650 struct mlx5_txq_ctrl *txq_ctrl; 651 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 652 653 txq_ctrl = mlx5_txq_get(dev, cur_queue); 654 if (txq_ctrl == NULL) { 655 rte_errno = EINVAL; 656 DRV_LOG(ERR, "Failed to get port %u Tx queue %d", 657 dev->data->port_id, cur_queue); 658 return -rte_errno; 659 } 660 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 661 rte_errno = EINVAL; 662 DRV_LOG(ERR, "port %u queue %d not a hairpin Txq", 663 dev->data->port_id, cur_queue); 664 mlx5_txq_release(dev, cur_queue); 665 return -rte_errno; 666 } 667 /* Already unbound, return success before obj checking. */ 668 if (txq_ctrl->hairpin_status == 0) { 669 DRV_LOG(DEBUG, "port %u Tx queue %d is already unbound", 670 dev->data->port_id, cur_queue); 671 mlx5_txq_release(dev, cur_queue); 672 return 0; 673 } 674 if (!txq_ctrl->obj || !txq_ctrl->obj->sq) { 675 rte_errno = ENOMEM; 676 DRV_LOG(ERR, "port %u no Txq object found: %d", 677 dev->data->port_id, cur_queue); 678 mlx5_txq_release(dev, cur_queue); 679 return -rte_errno; 680 } 681 sq_attr.state = MLX5_SQC_STATE_RST; 682 sq_attr.sq_state = MLX5_SQC_STATE_RST; 683 ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq, &sq_attr); 684 if (ret == 0) 685 txq_ctrl->hairpin_status = 0; 686 mlx5_txq_release(dev, cur_queue); 687 } else { 688 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, cur_queue); 689 struct mlx5_rxq_ctrl *rxq_ctrl; 690 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 691 692 if (rxq == NULL) { 693 rte_errno = EINVAL; 694 DRV_LOG(ERR, "Failed to get port %u Rx queue %d", 695 dev->data->port_id, cur_queue); 696 return -rte_errno; 697 } 698 rxq_ctrl = rxq->ctrl; 699 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) { 700 rte_errno = EINVAL; 701 DRV_LOG(ERR, "port %u queue %d not a hairpin Rxq", 702 dev->data->port_id, cur_queue); 703 return -rte_errno; 704 } 705 if (rxq->hairpin_status == 0) { 706 DRV_LOG(DEBUG, "port %u Rx queue %d is already unbound", 707 dev->data->port_id, cur_queue); 708 return 0; 709 } 710 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) { 711 rte_errno = ENOMEM; 712 DRV_LOG(ERR, "port %u no Rxq object found: %d", 713 dev->data->port_id, cur_queue); 714 return -rte_errno; 715 } 716 rq_attr.state = MLX5_SQC_STATE_RST; 717 rq_attr.rq_state = MLX5_SQC_STATE_RST; 718 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr); 719 if (ret == 0) 720 rxq->hairpin_status = 0; 721 } 722 return ret; 723 } 724 725 /* 726 * Bind the hairpin port pairs, from the Tx to the peer Rx. 727 * This function only supports to bind the Tx to one Rx. 728 * 729 * @param dev 730 * Pointer to Ethernet device structure. 731 * @param rx_port 732 * Port identifier of the Rx port. 733 * 734 * @return 735 * 0 on success, a negative errno value otherwise and rte_errno is set. 736 */ 737 static int 738 mlx5_hairpin_bind_single_port(struct rte_eth_dev *dev, uint16_t rx_port) 739 { 740 struct mlx5_priv *priv = dev->data->dev_private; 741 int ret = 0; 742 struct mlx5_txq_ctrl *txq_ctrl; 743 uint32_t i; 744 struct rte_hairpin_peer_info peer = {0xffffff}; 745 struct rte_hairpin_peer_info cur; 746 const struct rte_eth_hairpin_conf *conf; 747 uint16_t num_q = 0; 748 uint16_t local_port = priv->dev_data->port_id; 749 uint32_t manual; 750 uint32_t explicit; 751 uint16_t rx_queue; 752 753 if (mlx5_eth_find_next(rx_port, dev->device) != rx_port) { 754 rte_errno = ENODEV; 755 DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port); 756 return -rte_errno; 757 } 758 /* 759 * Before binding TxQ to peer RxQ, first round loop will be used for 760 * checking the queues' configuration consistency. This would be a 761 * little time consuming but better than doing the rollback. 762 */ 763 for (i = 0; i != priv->txqs_n; i++) { 764 txq_ctrl = mlx5_txq_get(dev, i); 765 if (txq_ctrl == NULL) 766 continue; 767 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 768 mlx5_txq_release(dev, i); 769 continue; 770 } 771 /* 772 * All hairpin Tx queues of a single port that connected to the 773 * same peer Rx port should have the same "auto binding" and 774 * "implicit Tx flow" modes. 775 * Peer consistency checking will be done in per queue binding. 776 */ 777 conf = &txq_ctrl->hairpin_conf; 778 if (conf->peers[0].port == rx_port) { 779 if (num_q == 0) { 780 manual = conf->manual_bind; 781 explicit = conf->tx_explicit; 782 } else { 783 if (manual != conf->manual_bind || 784 explicit != conf->tx_explicit) { 785 rte_errno = EINVAL; 786 DRV_LOG(ERR, "port %u queue %d mode" 787 " mismatch: %u %u, %u %u", 788 local_port, i, manual, 789 conf->manual_bind, explicit, 790 conf->tx_explicit); 791 mlx5_txq_release(dev, i); 792 return -rte_errno; 793 } 794 } 795 num_q++; 796 } 797 mlx5_txq_release(dev, i); 798 } 799 /* Once no queue is configured, success is returned directly. */ 800 if (num_q == 0) 801 return ret; 802 /* All the hairpin TX queues need to be traversed again. */ 803 for (i = 0; i != priv->txqs_n; i++) { 804 txq_ctrl = mlx5_txq_get(dev, i); 805 if (txq_ctrl == NULL) 806 continue; 807 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 808 mlx5_txq_release(dev, i); 809 continue; 810 } 811 if (txq_ctrl->hairpin_conf.peers[0].port != rx_port) { 812 mlx5_txq_release(dev, i); 813 continue; 814 } 815 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue; 816 /* 817 * Fetch peer RxQ's information. 818 * No need to pass the information of the current queue. 819 */ 820 ret = rte_eth_hairpin_queue_peer_update(rx_port, rx_queue, 821 NULL, &peer, 1); 822 if (ret != 0) { 823 mlx5_txq_release(dev, i); 824 goto error; 825 } 826 /* Accessing its own device, inside mlx5 PMD. */ 827 ret = mlx5_hairpin_queue_peer_bind(dev, i, &peer, 1); 828 if (ret != 0) { 829 mlx5_txq_release(dev, i); 830 goto error; 831 } 832 /* Pass TxQ's information to peer RxQ and try binding. */ 833 cur.peer_q = rx_queue; 834 cur.qp_id = txq_ctrl->obj->sq->id; 835 cur.vhca_id = priv->config.hca_attr.vhca_id; 836 cur.tx_explicit = txq_ctrl->hairpin_conf.tx_explicit; 837 cur.manual_bind = txq_ctrl->hairpin_conf.manual_bind; 838 /* 839 * In order to access another device in a proper way, RTE level 840 * private function is needed. 841 */ 842 ret = rte_eth_hairpin_queue_peer_bind(rx_port, rx_queue, 843 &cur, 0); 844 if (ret != 0) { 845 mlx5_txq_release(dev, i); 846 goto error; 847 } 848 mlx5_txq_release(dev, i); 849 } 850 return 0; 851 error: 852 /* 853 * Do roll-back process for the queues already bound. 854 * No need to check the return value of the queue unbind function. 855 */ 856 do { 857 /* No validation is needed here. */ 858 txq_ctrl = mlx5_txq_get(dev, i); 859 if (txq_ctrl == NULL) 860 continue; 861 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue; 862 rte_eth_hairpin_queue_peer_unbind(rx_port, rx_queue, 0); 863 mlx5_hairpin_queue_peer_unbind(dev, i, 1); 864 mlx5_txq_release(dev, i); 865 } while (i--); 866 return ret; 867 } 868 869 /* 870 * Unbind the hairpin port pair, HW configuration of both devices will be clear 871 * and status will be reset for all the queues used between the them. 872 * This function only supports to unbind the Tx from one Rx. 873 * 874 * @param dev 875 * Pointer to Ethernet device structure. 876 * @param rx_port 877 * Port identifier of the Rx port. 878 * 879 * @return 880 * 0 on success, a negative errno value otherwise and rte_errno is set. 881 */ 882 static int 883 mlx5_hairpin_unbind_single_port(struct rte_eth_dev *dev, uint16_t rx_port) 884 { 885 struct mlx5_priv *priv = dev->data->dev_private; 886 struct mlx5_txq_ctrl *txq_ctrl; 887 uint32_t i; 888 int ret; 889 uint16_t cur_port = priv->dev_data->port_id; 890 891 if (mlx5_eth_find_next(rx_port, dev->device) != rx_port) { 892 rte_errno = ENODEV; 893 DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port); 894 return -rte_errno; 895 } 896 for (i = 0; i != priv->txqs_n; i++) { 897 uint16_t rx_queue; 898 899 txq_ctrl = mlx5_txq_get(dev, i); 900 if (txq_ctrl == NULL) 901 continue; 902 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 903 mlx5_txq_release(dev, i); 904 continue; 905 } 906 if (txq_ctrl->hairpin_conf.peers[0].port != rx_port) { 907 mlx5_txq_release(dev, i); 908 continue; 909 } 910 /* Indeed, only the first used queue needs to be checked. */ 911 if (txq_ctrl->hairpin_conf.manual_bind == 0) { 912 if (cur_port != rx_port) { 913 rte_errno = EINVAL; 914 DRV_LOG(ERR, "port %u and port %u are in" 915 " auto-bind mode", cur_port, rx_port); 916 mlx5_txq_release(dev, i); 917 return -rte_errno; 918 } else { 919 return 0; 920 } 921 } 922 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue; 923 mlx5_txq_release(dev, i); 924 ret = rte_eth_hairpin_queue_peer_unbind(rx_port, rx_queue, 0); 925 if (ret) { 926 DRV_LOG(ERR, "port %u Rx queue %d unbind - failure", 927 rx_port, rx_queue); 928 return ret; 929 } 930 ret = mlx5_hairpin_queue_peer_unbind(dev, i, 1); 931 if (ret) { 932 DRV_LOG(ERR, "port %u Tx queue %d unbind - failure", 933 cur_port, i); 934 return ret; 935 } 936 } 937 return 0; 938 } 939 940 /* 941 * Bind hairpin ports, Rx could be all ports when using RTE_MAX_ETHPORTS. 942 * @see mlx5_hairpin_bind_single_port() 943 */ 944 int 945 mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port) 946 { 947 int ret = 0; 948 uint16_t p, pp; 949 950 /* 951 * If the Rx port has no hairpin configuration with the current port, 952 * the binding will be skipped in the called function of single port. 953 * Device started status will be checked only before the queue 954 * information updating. 955 */ 956 if (rx_port == RTE_MAX_ETHPORTS) { 957 MLX5_ETH_FOREACH_DEV(p, dev->device) { 958 ret = mlx5_hairpin_bind_single_port(dev, p); 959 if (ret != 0) 960 goto unbind; 961 } 962 return ret; 963 } else { 964 return mlx5_hairpin_bind_single_port(dev, rx_port); 965 } 966 unbind: 967 MLX5_ETH_FOREACH_DEV(pp, dev->device) 968 if (pp < p) 969 mlx5_hairpin_unbind_single_port(dev, pp); 970 return ret; 971 } 972 973 /* 974 * Unbind hairpin ports, Rx could be all ports when using RTE_MAX_ETHPORTS. 975 * @see mlx5_hairpin_unbind_single_port() 976 */ 977 int 978 mlx5_hairpin_unbind(struct rte_eth_dev *dev, uint16_t rx_port) 979 { 980 int ret = 0; 981 uint16_t p; 982 983 if (rx_port == RTE_MAX_ETHPORTS) 984 MLX5_ETH_FOREACH_DEV(p, dev->device) { 985 ret = mlx5_hairpin_unbind_single_port(dev, p); 986 if (ret != 0) 987 return ret; 988 } 989 else 990 ret = mlx5_hairpin_unbind_single_port(dev, rx_port); 991 return ret; 992 } 993 994 /* 995 * DPDK callback to get the hairpin peer ports list. 996 * This will return the actual number of peer ports and save the identifiers 997 * into the array (sorted, may be different from that when setting up the 998 * hairpin peer queues). 999 * The peer port ID could be the same as the port ID of the current device. 1000 * 1001 * @param dev 1002 * Pointer to Ethernet device structure. 1003 * @param peer_ports 1004 * Pointer to array to save the port identifiers. 1005 * @param len 1006 * The length of the array. 1007 * @param direction 1008 * Current port to peer port direction. 1009 * positive - current used as Tx to get all peer Rx ports. 1010 * zero - current used as Rx to get all peer Tx ports. 1011 * 1012 * @return 1013 * 0 or positive value on success, actual number of peer ports. 1014 * a negative errno value otherwise and rte_errno is set. 1015 */ 1016 int 1017 mlx5_hairpin_get_peer_ports(struct rte_eth_dev *dev, uint16_t *peer_ports, 1018 size_t len, uint32_t direction) 1019 { 1020 struct mlx5_priv *priv = dev->data->dev_private; 1021 struct mlx5_txq_ctrl *txq_ctrl; 1022 uint32_t i; 1023 uint16_t pp; 1024 uint32_t bits[(RTE_MAX_ETHPORTS + 31) / 32] = {0}; 1025 int ret = 0; 1026 1027 if (direction) { 1028 for (i = 0; i < priv->txqs_n; i++) { 1029 txq_ctrl = mlx5_txq_get(dev, i); 1030 if (!txq_ctrl) 1031 continue; 1032 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 1033 mlx5_txq_release(dev, i); 1034 continue; 1035 } 1036 pp = txq_ctrl->hairpin_conf.peers[0].port; 1037 if (pp >= RTE_MAX_ETHPORTS) { 1038 rte_errno = ERANGE; 1039 mlx5_txq_release(dev, i); 1040 DRV_LOG(ERR, "port %hu queue %u peer port " 1041 "out of range %hu", 1042 priv->dev_data->port_id, i, pp); 1043 return -rte_errno; 1044 } 1045 bits[pp / 32] |= 1 << (pp % 32); 1046 mlx5_txq_release(dev, i); 1047 } 1048 } else { 1049 for (i = 0; i < priv->rxqs_n; i++) { 1050 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1051 struct mlx5_rxq_ctrl *rxq_ctrl; 1052 1053 if (rxq == NULL) 1054 continue; 1055 rxq_ctrl = rxq->ctrl; 1056 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) 1057 continue; 1058 pp = rxq->hairpin_conf.peers[0].port; 1059 if (pp >= RTE_MAX_ETHPORTS) { 1060 rte_errno = ERANGE; 1061 DRV_LOG(ERR, "port %hu queue %u peer port " 1062 "out of range %hu", 1063 priv->dev_data->port_id, i, pp); 1064 return -rte_errno; 1065 } 1066 bits[pp / 32] |= 1 << (pp % 32); 1067 } 1068 } 1069 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 1070 if (bits[i / 32] & (1 << (i % 32))) { 1071 if ((size_t)ret >= len) { 1072 rte_errno = E2BIG; 1073 return -rte_errno; 1074 } 1075 peer_ports[ret++] = i; 1076 } 1077 } 1078 return ret; 1079 } 1080 1081 /** 1082 * DPDK callback to start the device. 1083 * 1084 * Simulate device start by attaching all configured flows. 1085 * 1086 * @param dev 1087 * Pointer to Ethernet device structure. 1088 * 1089 * @return 1090 * 0 on success, a negative errno value otherwise and rte_errno is set. 1091 */ 1092 int 1093 mlx5_dev_start(struct rte_eth_dev *dev) 1094 { 1095 struct mlx5_priv *priv = dev->data->dev_private; 1096 int ret; 1097 int fine_inline; 1098 1099 DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); 1100 fine_inline = rte_mbuf_dynflag_lookup 1101 (RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, NULL); 1102 if (fine_inline >= 0) 1103 rte_net_mlx5_dynf_inline_mask = 1UL << fine_inline; 1104 else 1105 rte_net_mlx5_dynf_inline_mask = 0; 1106 if (dev->data->nb_rx_queues > 0) { 1107 ret = mlx5_dev_configure_rss_reta(dev); 1108 if (ret) { 1109 DRV_LOG(ERR, "port %u reta config failed: %s", 1110 dev->data->port_id, strerror(rte_errno)); 1111 return -rte_errno; 1112 } 1113 } 1114 ret = mlx5_txpp_start(dev); 1115 if (ret) { 1116 DRV_LOG(ERR, "port %u Tx packet pacing init failed: %s", 1117 dev->data->port_id, strerror(rte_errno)); 1118 goto error; 1119 } 1120 if ((priv->sh->devx && priv->config.dv_flow_en && 1121 priv->config.dest_tir) && priv->obj_ops.lb_dummy_queue_create) { 1122 ret = priv->obj_ops.lb_dummy_queue_create(dev); 1123 if (ret) 1124 goto error; 1125 } 1126 ret = mlx5_txq_start(dev); 1127 if (ret) { 1128 DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", 1129 dev->data->port_id, strerror(rte_errno)); 1130 goto error; 1131 } 1132 if (priv->config.std_delay_drop || priv->config.hp_delay_drop) { 1133 if (!priv->config.vf && !priv->config.sf && 1134 !priv->representor) { 1135 ret = mlx5_get_flag_dropless_rq(dev); 1136 if (ret < 0) 1137 DRV_LOG(WARNING, 1138 "port %u cannot query dropless flag", 1139 dev->data->port_id); 1140 else if (!ret) 1141 DRV_LOG(WARNING, 1142 "port %u dropless_rq OFF, no rearming", 1143 dev->data->port_id); 1144 } else { 1145 DRV_LOG(DEBUG, 1146 "port %u doesn't support dropless_rq flag", 1147 dev->data->port_id); 1148 } 1149 } 1150 ret = mlx5_rxq_start(dev); 1151 if (ret) { 1152 DRV_LOG(ERR, "port %u Rx queue allocation failed: %s", 1153 dev->data->port_id, strerror(rte_errno)); 1154 goto error; 1155 } 1156 /* 1157 * Such step will be skipped if there is no hairpin TX queue configured 1158 * with RX peer queue from the same device. 1159 */ 1160 ret = mlx5_hairpin_auto_bind(dev); 1161 if (ret) { 1162 DRV_LOG(ERR, "port %u hairpin auto binding failed: %s", 1163 dev->data->port_id, strerror(rte_errno)); 1164 goto error; 1165 } 1166 /* Set started flag here for the following steps like control flow. */ 1167 dev->data->dev_started = 1; 1168 ret = mlx5_rx_intr_vec_enable(dev); 1169 if (ret) { 1170 DRV_LOG(ERR, "port %u Rx interrupt vector creation failed", 1171 dev->data->port_id); 1172 goto error; 1173 } 1174 mlx5_os_stats_init(dev); 1175 ret = mlx5_traffic_enable(dev); 1176 if (ret) { 1177 DRV_LOG(ERR, "port %u failed to set defaults flows", 1178 dev->data->port_id); 1179 goto error; 1180 } 1181 /* Set a mask and offset of dynamic metadata flows into Rx queues. */ 1182 mlx5_flow_rxq_dynf_metadata_set(dev); 1183 /* Set flags and context to convert Rx timestamps. */ 1184 mlx5_rxq_timestamp_set(dev); 1185 /* Set a mask and offset of scheduling on timestamp into Tx queues. */ 1186 mlx5_txq_dynf_timestamp_set(dev); 1187 /* Attach indirection table objects detached on port stop. */ 1188 ret = mlx5_action_handle_attach(dev); 1189 if (ret) { 1190 DRV_LOG(ERR, 1191 "port %u failed to attach indirect actions: %s", 1192 dev->data->port_id, rte_strerror(rte_errno)); 1193 goto error; 1194 } 1195 /* 1196 * In non-cached mode, it only needs to start the default mreg copy 1197 * action and no flow created by application exists anymore. 1198 * But it is worth wrapping the interface for further usage. 1199 */ 1200 ret = mlx5_flow_start_default(dev); 1201 if (ret) { 1202 DRV_LOG(DEBUG, "port %u failed to start default actions: %s", 1203 dev->data->port_id, strerror(rte_errno)); 1204 goto error; 1205 } 1206 if (mlx5_dev_ctx_shared_mempool_subscribe(dev) != 0) { 1207 DRV_LOG(ERR, "port %u failed to subscribe for mempool life cycle: %s", 1208 dev->data->port_id, rte_strerror(rte_errno)); 1209 goto error; 1210 } 1211 rte_wmb(); 1212 dev->tx_pkt_burst = mlx5_select_tx_function(dev); 1213 dev->rx_pkt_burst = mlx5_select_rx_function(dev); 1214 /* Enable datapath on secondary process. */ 1215 mlx5_mp_os_req_start_rxtx(dev); 1216 if (rte_intr_fd_get(priv->sh->intr_handle) >= 0) { 1217 priv->sh->port[priv->dev_port - 1].ih_port_id = 1218 (uint32_t)dev->data->port_id; 1219 } else { 1220 DRV_LOG(INFO, "port %u starts without LSC and RMV interrupts.", 1221 dev->data->port_id); 1222 dev->data->dev_conf.intr_conf.lsc = 0; 1223 dev->data->dev_conf.intr_conf.rmv = 0; 1224 } 1225 if (rte_intr_fd_get(priv->sh->intr_handle_devx) >= 0) 1226 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = 1227 (uint32_t)dev->data->port_id; 1228 return 0; 1229 error: 1230 ret = rte_errno; /* Save rte_errno before cleanup. */ 1231 /* Rollback. */ 1232 dev->data->dev_started = 0; 1233 mlx5_flow_stop_default(dev); 1234 mlx5_traffic_disable(dev); 1235 mlx5_txq_stop(dev); 1236 mlx5_rxq_stop(dev); 1237 if (priv->obj_ops.lb_dummy_queue_release) 1238 priv->obj_ops.lb_dummy_queue_release(dev); 1239 mlx5_txpp_stop(dev); /* Stop last. */ 1240 rte_errno = ret; /* Restore rte_errno. */ 1241 return -rte_errno; 1242 } 1243 1244 /** 1245 * DPDK callback to stop the device. 1246 * 1247 * Simulate device stop by detaching all configured flows. 1248 * 1249 * @param dev 1250 * Pointer to Ethernet device structure. 1251 */ 1252 int 1253 mlx5_dev_stop(struct rte_eth_dev *dev) 1254 { 1255 struct mlx5_priv *priv = dev->data->dev_private; 1256 1257 dev->data->dev_started = 0; 1258 /* Prevent crashes when queues are still in use. */ 1259 dev->rx_pkt_burst = removed_rx_burst; 1260 dev->tx_pkt_burst = removed_tx_burst; 1261 rte_wmb(); 1262 /* Disable datapath on secondary process. */ 1263 mlx5_mp_os_req_stop_rxtx(dev); 1264 rte_delay_us_sleep(1000 * priv->rxqs_n); 1265 DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); 1266 mlx5_flow_stop_default(dev); 1267 /* Control flows for default traffic can be removed firstly. */ 1268 mlx5_traffic_disable(dev); 1269 /* All RX queue flags will be cleared in the flush interface. */ 1270 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 1271 mlx5_flow_meter_rxq_flush(dev); 1272 mlx5_action_handle_detach(dev); 1273 mlx5_rx_intr_vec_disable(dev); 1274 priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS; 1275 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = RTE_MAX_ETHPORTS; 1276 mlx5_txq_stop(dev); 1277 mlx5_rxq_stop(dev); 1278 if (priv->obj_ops.lb_dummy_queue_release) 1279 priv->obj_ops.lb_dummy_queue_release(dev); 1280 mlx5_txpp_stop(dev); 1281 1282 return 0; 1283 } 1284 1285 /** 1286 * Enable traffic flows configured by control plane 1287 * 1288 * @param dev 1289 * Pointer to Ethernet device private data. 1290 * @param dev 1291 * Pointer to Ethernet device structure. 1292 * 1293 * @return 1294 * 0 on success, a negative errno value otherwise and rte_errno is set. 1295 */ 1296 int 1297 mlx5_traffic_enable(struct rte_eth_dev *dev) 1298 { 1299 struct mlx5_priv *priv = dev->data->dev_private; 1300 struct rte_flow_item_eth bcast = { 1301 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1302 }; 1303 struct rte_flow_item_eth ipv6_multi_spec = { 1304 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", 1305 }; 1306 struct rte_flow_item_eth ipv6_multi_mask = { 1307 .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00", 1308 }; 1309 struct rte_flow_item_eth unicast = { 1310 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1311 }; 1312 struct rte_flow_item_eth unicast_mask = { 1313 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1314 }; 1315 const unsigned int vlan_filter_n = priv->vlan_filter_n; 1316 const struct rte_ether_addr cmp = { 1317 .addr_bytes = "\x00\x00\x00\x00\x00\x00", 1318 }; 1319 unsigned int i; 1320 unsigned int j; 1321 int ret; 1322 1323 /* 1324 * Hairpin txq default flow should be created no matter if it is 1325 * isolation mode. Or else all the packets to be sent will be sent 1326 * out directly without the TX flow actions, e.g. encapsulation. 1327 */ 1328 for (i = 0; i != priv->txqs_n; ++i) { 1329 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 1330 if (!txq_ctrl) 1331 continue; 1332 /* Only Tx implicit mode requires the default Tx flow. */ 1333 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN && 1334 txq_ctrl->hairpin_conf.tx_explicit == 0 && 1335 txq_ctrl->hairpin_conf.peers[0].port == 1336 priv->dev_data->port_id) { 1337 ret = mlx5_ctrl_flow_source_queue(dev, i); 1338 if (ret) { 1339 mlx5_txq_release(dev, i); 1340 goto error; 1341 } 1342 } 1343 if ((priv->representor || priv->master) && 1344 priv->config.dv_esw_en) { 1345 if (mlx5_flow_create_devx_sq_miss_flow(dev, i) == 0) { 1346 DRV_LOG(ERR, 1347 "Port %u Tx queue %u SQ create representor devx default miss rule failed.", 1348 dev->data->port_id, i); 1349 goto error; 1350 } 1351 } 1352 mlx5_txq_release(dev, i); 1353 } 1354 if ((priv->master || priv->representor) && priv->config.dv_esw_en) { 1355 if (mlx5_flow_create_esw_table_zero_flow(dev)) 1356 priv->fdb_def_rule = 1; 1357 else 1358 DRV_LOG(INFO, "port %u FDB default rule cannot be" 1359 " configured - only Eswitch group 0 flows are" 1360 " supported.", dev->data->port_id); 1361 } 1362 if (!priv->config.lacp_by_user && priv->pf_bond >= 0) { 1363 ret = mlx5_flow_lacp_miss(dev); 1364 if (ret) 1365 DRV_LOG(INFO, "port %u LACP rule cannot be created - " 1366 "forward LACP to kernel.", dev->data->port_id); 1367 else 1368 DRV_LOG(INFO, "LACP traffic will be missed in port %u." 1369 , dev->data->port_id); 1370 } 1371 if (priv->isolated) 1372 return 0; 1373 if (dev->data->promiscuous) { 1374 struct rte_flow_item_eth promisc = { 1375 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1376 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1377 .type = 0, 1378 }; 1379 1380 ret = mlx5_ctrl_flow(dev, &promisc, &promisc); 1381 if (ret) 1382 goto error; 1383 } 1384 if (dev->data->all_multicast) { 1385 struct rte_flow_item_eth multicast = { 1386 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", 1387 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1388 .type = 0, 1389 }; 1390 1391 ret = mlx5_ctrl_flow(dev, &multicast, &multicast); 1392 if (ret) 1393 goto error; 1394 } else { 1395 /* Add broadcast/multicast flows. */ 1396 for (i = 0; i != vlan_filter_n; ++i) { 1397 uint16_t vlan = priv->vlan_filter[i]; 1398 1399 struct rte_flow_item_vlan vlan_spec = { 1400 .tci = rte_cpu_to_be_16(vlan), 1401 }; 1402 struct rte_flow_item_vlan vlan_mask = 1403 rte_flow_item_vlan_mask; 1404 1405 ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, 1406 &vlan_spec, &vlan_mask); 1407 if (ret) 1408 goto error; 1409 ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec, 1410 &ipv6_multi_mask, 1411 &vlan_spec, &vlan_mask); 1412 if (ret) 1413 goto error; 1414 } 1415 if (!vlan_filter_n) { 1416 ret = mlx5_ctrl_flow(dev, &bcast, &bcast); 1417 if (ret) 1418 goto error; 1419 ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec, 1420 &ipv6_multi_mask); 1421 if (ret) { 1422 /* Do not fail on IPv6 broadcast creation failure. */ 1423 DRV_LOG(WARNING, 1424 "IPv6 broadcast is not supported"); 1425 ret = 0; 1426 } 1427 } 1428 } 1429 /* Add MAC address flows. */ 1430 for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) { 1431 struct rte_ether_addr *mac = &dev->data->mac_addrs[i]; 1432 1433 if (!memcmp(mac, &cmp, sizeof(*mac))) 1434 continue; 1435 memcpy(&unicast.dst.addr_bytes, 1436 mac->addr_bytes, 1437 RTE_ETHER_ADDR_LEN); 1438 for (j = 0; j != vlan_filter_n; ++j) { 1439 uint16_t vlan = priv->vlan_filter[j]; 1440 1441 struct rte_flow_item_vlan vlan_spec = { 1442 .tci = rte_cpu_to_be_16(vlan), 1443 }; 1444 struct rte_flow_item_vlan vlan_mask = 1445 rte_flow_item_vlan_mask; 1446 1447 ret = mlx5_ctrl_flow_vlan(dev, &unicast, 1448 &unicast_mask, 1449 &vlan_spec, 1450 &vlan_mask); 1451 if (ret) 1452 goto error; 1453 } 1454 if (!vlan_filter_n) { 1455 ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask); 1456 if (ret) 1457 goto error; 1458 } 1459 } 1460 return 0; 1461 error: 1462 ret = rte_errno; /* Save rte_errno before cleanup. */ 1463 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false); 1464 rte_errno = ret; /* Restore rte_errno. */ 1465 return -rte_errno; 1466 } 1467 1468 1469 /** 1470 * Disable traffic flows configured by control plane 1471 * 1472 * @param dev 1473 * Pointer to Ethernet device private data. 1474 */ 1475 void 1476 mlx5_traffic_disable(struct rte_eth_dev *dev) 1477 { 1478 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false); 1479 } 1480 1481 /** 1482 * Restart traffic flows configured by control plane 1483 * 1484 * @param dev 1485 * Pointer to Ethernet device private data. 1486 * 1487 * @return 1488 * 0 on success, a negative errno value otherwise and rte_errno is set. 1489 */ 1490 int 1491 mlx5_traffic_restart(struct rte_eth_dev *dev) 1492 { 1493 if (dev->data->dev_started) { 1494 mlx5_traffic_disable(dev); 1495 return mlx5_traffic_enable(dev); 1496 } 1497 return 0; 1498 } 1499