1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <unistd.h> 7 8 #include <rte_ether.h> 9 #include <ethdev_driver.h> 10 #include <rte_interrupts.h> 11 #include <rte_alarm.h> 12 #include <rte_cycles.h> 13 14 #include <mlx5_malloc.h> 15 16 #include "mlx5.h" 17 #include "mlx5_flow.h" 18 #include "mlx5_rx.h" 19 #include "mlx5_tx.h" 20 #include "mlx5_utils.h" 21 #include "rte_pmd_mlx5.h" 22 23 /** 24 * Stop traffic on Tx queues. 25 * 26 * @param dev 27 * Pointer to Ethernet device structure. 28 */ 29 static void 30 mlx5_txq_stop(struct rte_eth_dev *dev) 31 { 32 struct mlx5_priv *priv = dev->data->dev_private; 33 unsigned int i; 34 35 for (i = 0; i != priv->txqs_n; ++i) 36 mlx5_txq_release(dev, i); 37 } 38 39 /** 40 * Start traffic on Tx queues. 41 * 42 * @param dev 43 * Pointer to Ethernet device structure. 44 * 45 * @return 46 * 0 on success, a negative errno value otherwise and rte_errno is set. 47 */ 48 static int 49 mlx5_txq_start(struct rte_eth_dev *dev) 50 { 51 struct mlx5_priv *priv = dev->data->dev_private; 52 unsigned int i; 53 int ret; 54 55 for (i = 0; i != priv->txqs_n; ++i) { 56 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 57 struct mlx5_txq_data *txq_data = &txq_ctrl->txq; 58 uint32_t flags = MLX5_MEM_RTE | MLX5_MEM_ZERO; 59 60 if (!txq_ctrl) 61 continue; 62 if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD) 63 txq_alloc_elts(txq_ctrl); 64 MLX5_ASSERT(!txq_ctrl->obj); 65 txq_ctrl->obj = mlx5_malloc(flags, sizeof(struct mlx5_txq_obj), 66 0, txq_ctrl->socket); 67 if (!txq_ctrl->obj) { 68 DRV_LOG(ERR, "Port %u Tx queue %u cannot allocate " 69 "memory resources.", dev->data->port_id, 70 txq_data->idx); 71 rte_errno = ENOMEM; 72 goto error; 73 } 74 ret = priv->obj_ops.txq_obj_new(dev, i); 75 if (ret < 0) { 76 mlx5_free(txq_ctrl->obj); 77 txq_ctrl->obj = NULL; 78 goto error; 79 } 80 if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD) { 81 size_t size = txq_data->cqe_s * sizeof(*txq_data->fcqs); 82 83 txq_data->fcqs = mlx5_malloc(flags, size, 84 RTE_CACHE_LINE_SIZE, 85 txq_ctrl->socket); 86 if (!txq_data->fcqs) { 87 DRV_LOG(ERR, "Port %u Tx queue %u cannot " 88 "allocate memory (FCQ).", 89 dev->data->port_id, i); 90 rte_errno = ENOMEM; 91 goto error; 92 } 93 } 94 DRV_LOG(DEBUG, "Port %u txq %u updated with %p.", 95 dev->data->port_id, i, (void *)&txq_ctrl->obj); 96 LIST_INSERT_HEAD(&priv->txqsobj, txq_ctrl->obj, next); 97 } 98 return 0; 99 error: 100 ret = rte_errno; /* Save rte_errno before cleanup. */ 101 do { 102 mlx5_txq_release(dev, i); 103 } while (i-- != 0); 104 rte_errno = ret; /* Restore rte_errno. */ 105 return -rte_errno; 106 } 107 108 /** 109 * Translate the chunk address to MR key in order to put in into the cache. 110 */ 111 static void 112 mlx5_rxq_mempool_register_cb(struct rte_mempool *mp, void *opaque, 113 struct rte_mempool_memhdr *memhdr, 114 unsigned int idx) 115 { 116 struct mlx5_rxq_data *rxq = opaque; 117 118 RTE_SET_USED(mp); 119 RTE_SET_USED(idx); 120 mlx5_rx_addr2mr(rxq, (uintptr_t)memhdr->addr); 121 } 122 123 /** 124 * Register Rx queue mempools and fill the Rx queue cache. 125 * This function tolerates repeated mempool registration. 126 * 127 * @param[in] rxq_ctrl 128 * Rx queue control data. 129 * 130 * @return 131 * 0 on success, (-1) on failure and rte_errno is set. 132 */ 133 static int 134 mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl) 135 { 136 struct mlx5_priv *priv = rxq_ctrl->priv; 137 struct rte_mempool *mp; 138 uint32_t s; 139 int ret = 0; 140 141 mlx5_mr_flush_local_cache(&rxq_ctrl->rxq.mr_ctrl); 142 /* MPRQ mempool is registered on creation, just fill the cache. */ 143 if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq)) { 144 rte_mempool_mem_iter(rxq_ctrl->rxq.mprq_mp, 145 mlx5_rxq_mempool_register_cb, 146 &rxq_ctrl->rxq); 147 return 0; 148 } 149 for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) { 150 mp = rxq_ctrl->rxq.rxseg[s].mp; 151 ret = mlx5_mr_mempool_register(&priv->sh->cdev->mr_scache, 152 priv->sh->cdev->pd, mp, 153 &priv->mp_id); 154 if (ret < 0 && rte_errno != EEXIST) 155 return ret; 156 rte_mempool_mem_iter(mp, mlx5_rxq_mempool_register_cb, 157 &rxq_ctrl->rxq); 158 } 159 return 0; 160 } 161 162 /** 163 * Stop traffic on Rx queues. 164 * 165 * @param dev 166 * Pointer to Ethernet device structure. 167 */ 168 static void 169 mlx5_rxq_stop(struct rte_eth_dev *dev) 170 { 171 struct mlx5_priv *priv = dev->data->dev_private; 172 unsigned int i; 173 174 for (i = 0; i != priv->rxqs_n; ++i) 175 mlx5_rxq_release(dev, i); 176 } 177 178 /** 179 * Start traffic on Rx queues. 180 * 181 * @param dev 182 * Pointer to Ethernet device structure. 183 * 184 * @return 185 * 0 on success, a negative errno value otherwise and rte_errno is set. 186 */ 187 static int 188 mlx5_rxq_start(struct rte_eth_dev *dev) 189 { 190 struct mlx5_priv *priv = dev->data->dev_private; 191 unsigned int i; 192 int ret = 0; 193 194 /* Allocate/reuse/resize mempool for Multi-Packet RQ. */ 195 if (mlx5_mprq_alloc_mp(dev)) { 196 /* Should not release Rx queues but return immediately. */ 197 return -rte_errno; 198 } 199 DRV_LOG(DEBUG, "Port %u device_attr.max_qp_wr is %d.", 200 dev->data->port_id, priv->sh->device_attr.max_qp_wr); 201 DRV_LOG(DEBUG, "Port %u device_attr.max_sge is %d.", 202 dev->data->port_id, priv->sh->device_attr.max_sge); 203 for (i = 0; i != priv->rxqs_n; ++i) { 204 struct mlx5_rxq_priv *rxq = mlx5_rxq_ref(dev, i); 205 struct mlx5_rxq_ctrl *rxq_ctrl; 206 207 if (rxq == NULL) 208 continue; 209 rxq_ctrl = rxq->ctrl; 210 if (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD) { 211 /* 212 * Pre-register the mempools. Regardless of whether 213 * the implicit registration is enabled or not, 214 * Rx mempool destruction is tracked to free MRs. 215 */ 216 if (mlx5_rxq_mempool_register(rxq_ctrl) < 0) 217 goto error; 218 ret = rxq_alloc_elts(rxq_ctrl); 219 if (ret) 220 goto error; 221 } 222 MLX5_ASSERT(!rxq_ctrl->obj); 223 rxq_ctrl->obj = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, 224 sizeof(*rxq_ctrl->obj), 0, 225 rxq_ctrl->socket); 226 if (!rxq_ctrl->obj) { 227 DRV_LOG(ERR, 228 "Port %u Rx queue %u can't allocate resources.", 229 dev->data->port_id, (*priv->rxqs)[i]->idx); 230 rte_errno = ENOMEM; 231 goto error; 232 } 233 ret = priv->obj_ops.rxq_obj_new(dev, i); 234 if (ret) { 235 mlx5_free(rxq_ctrl->obj); 236 rxq_ctrl->obj = NULL; 237 goto error; 238 } 239 DRV_LOG(DEBUG, "Port %u rxq %u updated with %p.", 240 dev->data->port_id, i, (void *)&rxq_ctrl->obj); 241 LIST_INSERT_HEAD(&priv->rxqsobj, rxq_ctrl->obj, next); 242 } 243 return 0; 244 error: 245 ret = rte_errno; /* Save rte_errno before cleanup. */ 246 do { 247 mlx5_rxq_release(dev, i); 248 } while (i-- != 0); 249 rte_errno = ret; /* Restore rte_errno. */ 250 return -rte_errno; 251 } 252 253 /** 254 * Binds Tx queues to Rx queues for hairpin. 255 * 256 * Binds Tx queues to the target Rx queues. 257 * 258 * @param dev 259 * Pointer to Ethernet device structure. 260 * 261 * @return 262 * 0 on success, a negative errno value otherwise and rte_errno is set. 263 */ 264 static int 265 mlx5_hairpin_auto_bind(struct rte_eth_dev *dev) 266 { 267 struct mlx5_priv *priv = dev->data->dev_private; 268 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 269 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 270 struct mlx5_txq_ctrl *txq_ctrl; 271 struct mlx5_rxq_priv *rxq; 272 struct mlx5_rxq_ctrl *rxq_ctrl; 273 struct mlx5_devx_obj *sq; 274 struct mlx5_devx_obj *rq; 275 unsigned int i; 276 int ret = 0; 277 bool need_auto = false; 278 uint16_t self_port = dev->data->port_id; 279 280 for (i = 0; i != priv->txqs_n; ++i) { 281 txq_ctrl = mlx5_txq_get(dev, i); 282 if (!txq_ctrl) 283 continue; 284 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN || 285 txq_ctrl->hairpin_conf.peers[0].port != self_port) { 286 mlx5_txq_release(dev, i); 287 continue; 288 } 289 if (txq_ctrl->hairpin_conf.manual_bind) { 290 mlx5_txq_release(dev, i); 291 return 0; 292 } 293 need_auto = true; 294 mlx5_txq_release(dev, i); 295 } 296 if (!need_auto) 297 return 0; 298 for (i = 0; i != priv->txqs_n; ++i) { 299 txq_ctrl = mlx5_txq_get(dev, i); 300 if (!txq_ctrl) 301 continue; 302 /* Skip hairpin queues with other peer ports. */ 303 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN || 304 txq_ctrl->hairpin_conf.peers[0].port != self_port) { 305 mlx5_txq_release(dev, i); 306 continue; 307 } 308 if (!txq_ctrl->obj) { 309 rte_errno = ENOMEM; 310 DRV_LOG(ERR, "port %u no txq object found: %d", 311 dev->data->port_id, i); 312 mlx5_txq_release(dev, i); 313 return -rte_errno; 314 } 315 sq = txq_ctrl->obj->sq; 316 rxq = mlx5_rxq_get(dev, txq_ctrl->hairpin_conf.peers[0].queue); 317 if (rxq == NULL) { 318 mlx5_txq_release(dev, i); 319 rte_errno = EINVAL; 320 DRV_LOG(ERR, "port %u no rxq object found: %d", 321 dev->data->port_id, 322 txq_ctrl->hairpin_conf.peers[0].queue); 323 return -rte_errno; 324 } 325 rxq_ctrl = rxq->ctrl; 326 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN || 327 rxq_ctrl->hairpin_conf.peers[0].queue != i) { 328 rte_errno = ENOMEM; 329 DRV_LOG(ERR, "port %u Tx queue %d can't be binded to " 330 "Rx queue %d", dev->data->port_id, 331 i, txq_ctrl->hairpin_conf.peers[0].queue); 332 goto error; 333 } 334 rq = rxq_ctrl->obj->rq; 335 if (!rq) { 336 rte_errno = ENOMEM; 337 DRV_LOG(ERR, "port %u hairpin no matching rxq: %d", 338 dev->data->port_id, 339 txq_ctrl->hairpin_conf.peers[0].queue); 340 goto error; 341 } 342 sq_attr.state = MLX5_SQC_STATE_RDY; 343 sq_attr.sq_state = MLX5_SQC_STATE_RST; 344 sq_attr.hairpin_peer_rq = rq->id; 345 sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 346 ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr); 347 if (ret) 348 goto error; 349 rq_attr.state = MLX5_SQC_STATE_RDY; 350 rq_attr.rq_state = MLX5_SQC_STATE_RST; 351 rq_attr.hairpin_peer_sq = sq->id; 352 rq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 353 ret = mlx5_devx_cmd_modify_rq(rq, &rq_attr); 354 if (ret) 355 goto error; 356 /* Qs with auto-bind will be destroyed directly. */ 357 rxq_ctrl->hairpin_status = 1; 358 txq_ctrl->hairpin_status = 1; 359 mlx5_txq_release(dev, i); 360 } 361 return 0; 362 error: 363 mlx5_txq_release(dev, i); 364 return -rte_errno; 365 } 366 367 /* 368 * Fetch the peer queue's SW & HW information. 369 * 370 * @param dev 371 * Pointer to Ethernet device structure. 372 * @param peer_queue 373 * Index of the queue to fetch the information. 374 * @param current_info 375 * Pointer to the input peer information, not used currently. 376 * @param peer_info 377 * Pointer to the structure to store the information, output. 378 * @param direction 379 * Positive to get the RxQ information, zero to get the TxQ information. 380 * 381 * @return 382 * 0 on success, a negative errno value otherwise and rte_errno is set. 383 */ 384 int 385 mlx5_hairpin_queue_peer_update(struct rte_eth_dev *dev, uint16_t peer_queue, 386 struct rte_hairpin_peer_info *current_info, 387 struct rte_hairpin_peer_info *peer_info, 388 uint32_t direction) 389 { 390 struct mlx5_priv *priv = dev->data->dev_private; 391 RTE_SET_USED(current_info); 392 393 if (dev->data->dev_started == 0) { 394 rte_errno = EBUSY; 395 DRV_LOG(ERR, "peer port %u is not started", 396 dev->data->port_id); 397 return -rte_errno; 398 } 399 /* 400 * Peer port used as egress. In the current design, hairpin Tx queue 401 * will be bound to the peer Rx queue. Indeed, only the information of 402 * peer Rx queue needs to be fetched. 403 */ 404 if (direction == 0) { 405 struct mlx5_txq_ctrl *txq_ctrl; 406 407 txq_ctrl = mlx5_txq_get(dev, peer_queue); 408 if (txq_ctrl == NULL) { 409 rte_errno = EINVAL; 410 DRV_LOG(ERR, "Failed to get port %u Tx queue %d", 411 dev->data->port_id, peer_queue); 412 return -rte_errno; 413 } 414 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 415 rte_errno = EINVAL; 416 DRV_LOG(ERR, "port %u queue %d is not a hairpin Txq", 417 dev->data->port_id, peer_queue); 418 mlx5_txq_release(dev, peer_queue); 419 return -rte_errno; 420 } 421 if (txq_ctrl->obj == NULL || txq_ctrl->obj->sq == NULL) { 422 rte_errno = ENOMEM; 423 DRV_LOG(ERR, "port %u no Txq object found: %d", 424 dev->data->port_id, peer_queue); 425 mlx5_txq_release(dev, peer_queue); 426 return -rte_errno; 427 } 428 peer_info->qp_id = txq_ctrl->obj->sq->id; 429 peer_info->vhca_id = priv->config.hca_attr.vhca_id; 430 /* 1-to-1 mapping, only the first one is used. */ 431 peer_info->peer_q = txq_ctrl->hairpin_conf.peers[0].queue; 432 peer_info->tx_explicit = txq_ctrl->hairpin_conf.tx_explicit; 433 peer_info->manual_bind = txq_ctrl->hairpin_conf.manual_bind; 434 mlx5_txq_release(dev, peer_queue); 435 } else { /* Peer port used as ingress. */ 436 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, peer_queue); 437 struct mlx5_rxq_ctrl *rxq_ctrl; 438 439 if (rxq == NULL) { 440 rte_errno = EINVAL; 441 DRV_LOG(ERR, "Failed to get port %u Rx queue %d", 442 dev->data->port_id, peer_queue); 443 return -rte_errno; 444 } 445 rxq_ctrl = rxq->ctrl; 446 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) { 447 rte_errno = EINVAL; 448 DRV_LOG(ERR, "port %u queue %d is not a hairpin Rxq", 449 dev->data->port_id, peer_queue); 450 return -rte_errno; 451 } 452 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) { 453 rte_errno = ENOMEM; 454 DRV_LOG(ERR, "port %u no Rxq object found: %d", 455 dev->data->port_id, peer_queue); 456 return -rte_errno; 457 } 458 peer_info->qp_id = rxq_ctrl->obj->rq->id; 459 peer_info->vhca_id = priv->config.hca_attr.vhca_id; 460 peer_info->peer_q = rxq_ctrl->hairpin_conf.peers[0].queue; 461 peer_info->tx_explicit = rxq_ctrl->hairpin_conf.tx_explicit; 462 peer_info->manual_bind = rxq_ctrl->hairpin_conf.manual_bind; 463 } 464 return 0; 465 } 466 467 /* 468 * Bind the hairpin queue with the peer HW information. 469 * This needs to be called twice both for Tx and Rx queues of a pair. 470 * If the queue is already bound, it is considered successful. 471 * 472 * @param dev 473 * Pointer to Ethernet device structure. 474 * @param cur_queue 475 * Index of the queue to change the HW configuration to bind. 476 * @param peer_info 477 * Pointer to information of the peer queue. 478 * @param direction 479 * Positive to configure the TxQ, zero to configure the RxQ. 480 * 481 * @return 482 * 0 on success, a negative errno value otherwise and rte_errno is set. 483 */ 484 int 485 mlx5_hairpin_queue_peer_bind(struct rte_eth_dev *dev, uint16_t cur_queue, 486 struct rte_hairpin_peer_info *peer_info, 487 uint32_t direction) 488 { 489 int ret = 0; 490 491 /* 492 * Consistency checking of the peer queue: opposite direction is used 493 * to get the peer queue info with ethdev port ID, no need to check. 494 */ 495 if (peer_info->peer_q != cur_queue) { 496 rte_errno = EINVAL; 497 DRV_LOG(ERR, "port %u queue %d and peer queue %d mismatch", 498 dev->data->port_id, cur_queue, peer_info->peer_q); 499 return -rte_errno; 500 } 501 if (direction != 0) { 502 struct mlx5_txq_ctrl *txq_ctrl; 503 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 504 505 txq_ctrl = mlx5_txq_get(dev, cur_queue); 506 if (txq_ctrl == NULL) { 507 rte_errno = EINVAL; 508 DRV_LOG(ERR, "Failed to get port %u Tx queue %d", 509 dev->data->port_id, cur_queue); 510 return -rte_errno; 511 } 512 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 513 rte_errno = EINVAL; 514 DRV_LOG(ERR, "port %u queue %d not a hairpin Txq", 515 dev->data->port_id, cur_queue); 516 mlx5_txq_release(dev, cur_queue); 517 return -rte_errno; 518 } 519 if (txq_ctrl->obj == NULL || txq_ctrl->obj->sq == NULL) { 520 rte_errno = ENOMEM; 521 DRV_LOG(ERR, "port %u no Txq object found: %d", 522 dev->data->port_id, cur_queue); 523 mlx5_txq_release(dev, cur_queue); 524 return -rte_errno; 525 } 526 if (txq_ctrl->hairpin_status != 0) { 527 DRV_LOG(DEBUG, "port %u Tx queue %d is already bound", 528 dev->data->port_id, cur_queue); 529 mlx5_txq_release(dev, cur_queue); 530 return 0; 531 } 532 /* 533 * All queues' of one port consistency checking is done in the 534 * bind() function, and that is optional. 535 */ 536 if (peer_info->tx_explicit != 537 txq_ctrl->hairpin_conf.tx_explicit) { 538 rte_errno = EINVAL; 539 DRV_LOG(ERR, "port %u Tx queue %d and peer Tx rule mode" 540 " mismatch", dev->data->port_id, cur_queue); 541 mlx5_txq_release(dev, cur_queue); 542 return -rte_errno; 543 } 544 if (peer_info->manual_bind != 545 txq_ctrl->hairpin_conf.manual_bind) { 546 rte_errno = EINVAL; 547 DRV_LOG(ERR, "port %u Tx queue %d and peer binding mode" 548 " mismatch", dev->data->port_id, cur_queue); 549 mlx5_txq_release(dev, cur_queue); 550 return -rte_errno; 551 } 552 sq_attr.state = MLX5_SQC_STATE_RDY; 553 sq_attr.sq_state = MLX5_SQC_STATE_RST; 554 sq_attr.hairpin_peer_rq = peer_info->qp_id; 555 sq_attr.hairpin_peer_vhca = peer_info->vhca_id; 556 ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq, &sq_attr); 557 if (ret == 0) 558 txq_ctrl->hairpin_status = 1; 559 mlx5_txq_release(dev, cur_queue); 560 } else { 561 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, cur_queue); 562 struct mlx5_rxq_ctrl *rxq_ctrl; 563 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 564 565 if (rxq == NULL) { 566 rte_errno = EINVAL; 567 DRV_LOG(ERR, "Failed to get port %u Rx queue %d", 568 dev->data->port_id, cur_queue); 569 return -rte_errno; 570 } 571 rxq_ctrl = rxq->ctrl; 572 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) { 573 rte_errno = EINVAL; 574 DRV_LOG(ERR, "port %u queue %d not a hairpin Rxq", 575 dev->data->port_id, cur_queue); 576 return -rte_errno; 577 } 578 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) { 579 rte_errno = ENOMEM; 580 DRV_LOG(ERR, "port %u no Rxq object found: %d", 581 dev->data->port_id, cur_queue); 582 return -rte_errno; 583 } 584 if (rxq_ctrl->hairpin_status != 0) { 585 DRV_LOG(DEBUG, "port %u Rx queue %d is already bound", 586 dev->data->port_id, cur_queue); 587 return 0; 588 } 589 if (peer_info->tx_explicit != 590 rxq_ctrl->hairpin_conf.tx_explicit) { 591 rte_errno = EINVAL; 592 DRV_LOG(ERR, "port %u Rx queue %d and peer Tx rule mode" 593 " mismatch", dev->data->port_id, cur_queue); 594 return -rte_errno; 595 } 596 if (peer_info->manual_bind != 597 rxq_ctrl->hairpin_conf.manual_bind) { 598 rte_errno = EINVAL; 599 DRV_LOG(ERR, "port %u Rx queue %d and peer binding mode" 600 " mismatch", dev->data->port_id, cur_queue); 601 return -rte_errno; 602 } 603 rq_attr.state = MLX5_SQC_STATE_RDY; 604 rq_attr.rq_state = MLX5_SQC_STATE_RST; 605 rq_attr.hairpin_peer_sq = peer_info->qp_id; 606 rq_attr.hairpin_peer_vhca = peer_info->vhca_id; 607 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr); 608 if (ret == 0) 609 rxq_ctrl->hairpin_status = 1; 610 } 611 return ret; 612 } 613 614 /* 615 * Unbind the hairpin queue and reset its HW configuration. 616 * This needs to be called twice both for Tx and Rx queues of a pair. 617 * If the queue is already unbound, it is considered successful. 618 * 619 * @param dev 620 * Pointer to Ethernet device structure. 621 * @param cur_queue 622 * Index of the queue to change the HW configuration to unbind. 623 * @param direction 624 * Positive to reset the TxQ, zero to reset the RxQ. 625 * 626 * @return 627 * 0 on success, a negative errno value otherwise and rte_errno is set. 628 */ 629 int 630 mlx5_hairpin_queue_peer_unbind(struct rte_eth_dev *dev, uint16_t cur_queue, 631 uint32_t direction) 632 { 633 int ret = 0; 634 635 if (direction != 0) { 636 struct mlx5_txq_ctrl *txq_ctrl; 637 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 638 639 txq_ctrl = mlx5_txq_get(dev, cur_queue); 640 if (txq_ctrl == NULL) { 641 rte_errno = EINVAL; 642 DRV_LOG(ERR, "Failed to get port %u Tx queue %d", 643 dev->data->port_id, cur_queue); 644 return -rte_errno; 645 } 646 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 647 rte_errno = EINVAL; 648 DRV_LOG(ERR, "port %u queue %d not a hairpin Txq", 649 dev->data->port_id, cur_queue); 650 mlx5_txq_release(dev, cur_queue); 651 return -rte_errno; 652 } 653 /* Already unbound, return success before obj checking. */ 654 if (txq_ctrl->hairpin_status == 0) { 655 DRV_LOG(DEBUG, "port %u Tx queue %d is already unbound", 656 dev->data->port_id, cur_queue); 657 mlx5_txq_release(dev, cur_queue); 658 return 0; 659 } 660 if (!txq_ctrl->obj || !txq_ctrl->obj->sq) { 661 rte_errno = ENOMEM; 662 DRV_LOG(ERR, "port %u no Txq object found: %d", 663 dev->data->port_id, cur_queue); 664 mlx5_txq_release(dev, cur_queue); 665 return -rte_errno; 666 } 667 sq_attr.state = MLX5_SQC_STATE_RST; 668 sq_attr.sq_state = MLX5_SQC_STATE_RST; 669 ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq, &sq_attr); 670 if (ret == 0) 671 txq_ctrl->hairpin_status = 0; 672 mlx5_txq_release(dev, cur_queue); 673 } else { 674 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, cur_queue); 675 struct mlx5_rxq_ctrl *rxq_ctrl; 676 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 677 678 if (rxq == NULL) { 679 rte_errno = EINVAL; 680 DRV_LOG(ERR, "Failed to get port %u Rx queue %d", 681 dev->data->port_id, cur_queue); 682 return -rte_errno; 683 } 684 rxq_ctrl = rxq->ctrl; 685 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) { 686 rte_errno = EINVAL; 687 DRV_LOG(ERR, "port %u queue %d not a hairpin Rxq", 688 dev->data->port_id, cur_queue); 689 return -rte_errno; 690 } 691 if (rxq_ctrl->hairpin_status == 0) { 692 DRV_LOG(DEBUG, "port %u Rx queue %d is already unbound", 693 dev->data->port_id, cur_queue); 694 return 0; 695 } 696 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) { 697 rte_errno = ENOMEM; 698 DRV_LOG(ERR, "port %u no Rxq object found: %d", 699 dev->data->port_id, cur_queue); 700 return -rte_errno; 701 } 702 rq_attr.state = MLX5_SQC_STATE_RST; 703 rq_attr.rq_state = MLX5_SQC_STATE_RST; 704 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr); 705 if (ret == 0) 706 rxq_ctrl->hairpin_status = 0; 707 } 708 return ret; 709 } 710 711 /* 712 * Bind the hairpin port pairs, from the Tx to the peer Rx. 713 * This function only supports to bind the Tx to one Rx. 714 * 715 * @param dev 716 * Pointer to Ethernet device structure. 717 * @param rx_port 718 * Port identifier of the Rx port. 719 * 720 * @return 721 * 0 on success, a negative errno value otherwise and rte_errno is set. 722 */ 723 static int 724 mlx5_hairpin_bind_single_port(struct rte_eth_dev *dev, uint16_t rx_port) 725 { 726 struct mlx5_priv *priv = dev->data->dev_private; 727 int ret = 0; 728 struct mlx5_txq_ctrl *txq_ctrl; 729 uint32_t i; 730 struct rte_hairpin_peer_info peer = {0xffffff}; 731 struct rte_hairpin_peer_info cur; 732 const struct rte_eth_hairpin_conf *conf; 733 uint16_t num_q = 0; 734 uint16_t local_port = priv->dev_data->port_id; 735 uint32_t manual; 736 uint32_t explicit; 737 uint16_t rx_queue; 738 739 if (mlx5_eth_find_next(rx_port, dev->device) != rx_port) { 740 rte_errno = ENODEV; 741 DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port); 742 return -rte_errno; 743 } 744 /* 745 * Before binding TxQ to peer RxQ, first round loop will be used for 746 * checking the queues' configuration consistency. This would be a 747 * little time consuming but better than doing the rollback. 748 */ 749 for (i = 0; i != priv->txqs_n; i++) { 750 txq_ctrl = mlx5_txq_get(dev, i); 751 if (txq_ctrl == NULL) 752 continue; 753 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 754 mlx5_txq_release(dev, i); 755 continue; 756 } 757 /* 758 * All hairpin Tx queues of a single port that connected to the 759 * same peer Rx port should have the same "auto binding" and 760 * "implicit Tx flow" modes. 761 * Peer consistency checking will be done in per queue binding. 762 */ 763 conf = &txq_ctrl->hairpin_conf; 764 if (conf->peers[0].port == rx_port) { 765 if (num_q == 0) { 766 manual = conf->manual_bind; 767 explicit = conf->tx_explicit; 768 } else { 769 if (manual != conf->manual_bind || 770 explicit != conf->tx_explicit) { 771 rte_errno = EINVAL; 772 DRV_LOG(ERR, "port %u queue %d mode" 773 " mismatch: %u %u, %u %u", 774 local_port, i, manual, 775 conf->manual_bind, explicit, 776 conf->tx_explicit); 777 mlx5_txq_release(dev, i); 778 return -rte_errno; 779 } 780 } 781 num_q++; 782 } 783 mlx5_txq_release(dev, i); 784 } 785 /* Once no queue is configured, success is returned directly. */ 786 if (num_q == 0) 787 return ret; 788 /* All the hairpin TX queues need to be traversed again. */ 789 for (i = 0; i != priv->txqs_n; i++) { 790 txq_ctrl = mlx5_txq_get(dev, i); 791 if (txq_ctrl == NULL) 792 continue; 793 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 794 mlx5_txq_release(dev, i); 795 continue; 796 } 797 if (txq_ctrl->hairpin_conf.peers[0].port != rx_port) { 798 mlx5_txq_release(dev, i); 799 continue; 800 } 801 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue; 802 /* 803 * Fetch peer RxQ's information. 804 * No need to pass the information of the current queue. 805 */ 806 ret = rte_eth_hairpin_queue_peer_update(rx_port, rx_queue, 807 NULL, &peer, 1); 808 if (ret != 0) { 809 mlx5_txq_release(dev, i); 810 goto error; 811 } 812 /* Accessing its own device, inside mlx5 PMD. */ 813 ret = mlx5_hairpin_queue_peer_bind(dev, i, &peer, 1); 814 if (ret != 0) { 815 mlx5_txq_release(dev, i); 816 goto error; 817 } 818 /* Pass TxQ's information to peer RxQ and try binding. */ 819 cur.peer_q = rx_queue; 820 cur.qp_id = txq_ctrl->obj->sq->id; 821 cur.vhca_id = priv->config.hca_attr.vhca_id; 822 cur.tx_explicit = txq_ctrl->hairpin_conf.tx_explicit; 823 cur.manual_bind = txq_ctrl->hairpin_conf.manual_bind; 824 /* 825 * In order to access another device in a proper way, RTE level 826 * private function is needed. 827 */ 828 ret = rte_eth_hairpin_queue_peer_bind(rx_port, rx_queue, 829 &cur, 0); 830 if (ret != 0) { 831 mlx5_txq_release(dev, i); 832 goto error; 833 } 834 mlx5_txq_release(dev, i); 835 } 836 return 0; 837 error: 838 /* 839 * Do roll-back process for the queues already bound. 840 * No need to check the return value of the queue unbind function. 841 */ 842 do { 843 /* No validation is needed here. */ 844 txq_ctrl = mlx5_txq_get(dev, i); 845 if (txq_ctrl == NULL) 846 continue; 847 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue; 848 rte_eth_hairpin_queue_peer_unbind(rx_port, rx_queue, 0); 849 mlx5_hairpin_queue_peer_unbind(dev, i, 1); 850 mlx5_txq_release(dev, i); 851 } while (i--); 852 return ret; 853 } 854 855 /* 856 * Unbind the hairpin port pair, HW configuration of both devices will be clear 857 * and status will be reset for all the queues used between the them. 858 * This function only supports to unbind the Tx from one Rx. 859 * 860 * @param dev 861 * Pointer to Ethernet device structure. 862 * @param rx_port 863 * Port identifier of the Rx port. 864 * 865 * @return 866 * 0 on success, a negative errno value otherwise and rte_errno is set. 867 */ 868 static int 869 mlx5_hairpin_unbind_single_port(struct rte_eth_dev *dev, uint16_t rx_port) 870 { 871 struct mlx5_priv *priv = dev->data->dev_private; 872 struct mlx5_txq_ctrl *txq_ctrl; 873 uint32_t i; 874 int ret; 875 uint16_t cur_port = priv->dev_data->port_id; 876 877 if (mlx5_eth_find_next(rx_port, dev->device) != rx_port) { 878 rte_errno = ENODEV; 879 DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port); 880 return -rte_errno; 881 } 882 for (i = 0; i != priv->txqs_n; i++) { 883 uint16_t rx_queue; 884 885 txq_ctrl = mlx5_txq_get(dev, i); 886 if (txq_ctrl == NULL) 887 continue; 888 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 889 mlx5_txq_release(dev, i); 890 continue; 891 } 892 if (txq_ctrl->hairpin_conf.peers[0].port != rx_port) { 893 mlx5_txq_release(dev, i); 894 continue; 895 } 896 /* Indeed, only the first used queue needs to be checked. */ 897 if (txq_ctrl->hairpin_conf.manual_bind == 0) { 898 if (cur_port != rx_port) { 899 rte_errno = EINVAL; 900 DRV_LOG(ERR, "port %u and port %u are in" 901 " auto-bind mode", cur_port, rx_port); 902 mlx5_txq_release(dev, i); 903 return -rte_errno; 904 } else { 905 return 0; 906 } 907 } 908 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue; 909 mlx5_txq_release(dev, i); 910 ret = rte_eth_hairpin_queue_peer_unbind(rx_port, rx_queue, 0); 911 if (ret) { 912 DRV_LOG(ERR, "port %u Rx queue %d unbind - failure", 913 rx_port, rx_queue); 914 return ret; 915 } 916 ret = mlx5_hairpin_queue_peer_unbind(dev, i, 1); 917 if (ret) { 918 DRV_LOG(ERR, "port %u Tx queue %d unbind - failure", 919 cur_port, i); 920 return ret; 921 } 922 } 923 return 0; 924 } 925 926 /* 927 * Bind hairpin ports, Rx could be all ports when using RTE_MAX_ETHPORTS. 928 * @see mlx5_hairpin_bind_single_port() 929 */ 930 int 931 mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port) 932 { 933 int ret = 0; 934 uint16_t p, pp; 935 936 /* 937 * If the Rx port has no hairpin configuration with the current port, 938 * the binding will be skipped in the called function of single port. 939 * Device started status will be checked only before the queue 940 * information updating. 941 */ 942 if (rx_port == RTE_MAX_ETHPORTS) { 943 MLX5_ETH_FOREACH_DEV(p, dev->device) { 944 ret = mlx5_hairpin_bind_single_port(dev, p); 945 if (ret != 0) 946 goto unbind; 947 } 948 return ret; 949 } else { 950 return mlx5_hairpin_bind_single_port(dev, rx_port); 951 } 952 unbind: 953 MLX5_ETH_FOREACH_DEV(pp, dev->device) 954 if (pp < p) 955 mlx5_hairpin_unbind_single_port(dev, pp); 956 return ret; 957 } 958 959 /* 960 * Unbind hairpin ports, Rx could be all ports when using RTE_MAX_ETHPORTS. 961 * @see mlx5_hairpin_unbind_single_port() 962 */ 963 int 964 mlx5_hairpin_unbind(struct rte_eth_dev *dev, uint16_t rx_port) 965 { 966 int ret = 0; 967 uint16_t p; 968 969 if (rx_port == RTE_MAX_ETHPORTS) 970 MLX5_ETH_FOREACH_DEV(p, dev->device) { 971 ret = mlx5_hairpin_unbind_single_port(dev, p); 972 if (ret != 0) 973 return ret; 974 } 975 else 976 ret = mlx5_hairpin_unbind_single_port(dev, rx_port); 977 return ret; 978 } 979 980 /* 981 * DPDK callback to get the hairpin peer ports list. 982 * This will return the actual number of peer ports and save the identifiers 983 * into the array (sorted, may be different from that when setting up the 984 * hairpin peer queues). 985 * The peer port ID could be the same as the port ID of the current device. 986 * 987 * @param dev 988 * Pointer to Ethernet device structure. 989 * @param peer_ports 990 * Pointer to array to save the port identifiers. 991 * @param len 992 * The length of the array. 993 * @param direction 994 * Current port to peer port direction. 995 * positive - current used as Tx to get all peer Rx ports. 996 * zero - current used as Rx to get all peer Tx ports. 997 * 998 * @return 999 * 0 or positive value on success, actual number of peer ports. 1000 * a negative errno value otherwise and rte_errno is set. 1001 */ 1002 int 1003 mlx5_hairpin_get_peer_ports(struct rte_eth_dev *dev, uint16_t *peer_ports, 1004 size_t len, uint32_t direction) 1005 { 1006 struct mlx5_priv *priv = dev->data->dev_private; 1007 struct mlx5_txq_ctrl *txq_ctrl; 1008 uint32_t i; 1009 uint16_t pp; 1010 uint32_t bits[(RTE_MAX_ETHPORTS + 31) / 32] = {0}; 1011 int ret = 0; 1012 1013 if (direction) { 1014 for (i = 0; i < priv->txqs_n; i++) { 1015 txq_ctrl = mlx5_txq_get(dev, i); 1016 if (!txq_ctrl) 1017 continue; 1018 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 1019 mlx5_txq_release(dev, i); 1020 continue; 1021 } 1022 pp = txq_ctrl->hairpin_conf.peers[0].port; 1023 if (pp >= RTE_MAX_ETHPORTS) { 1024 rte_errno = ERANGE; 1025 mlx5_txq_release(dev, i); 1026 DRV_LOG(ERR, "port %hu queue %u peer port " 1027 "out of range %hu", 1028 priv->dev_data->port_id, i, pp); 1029 return -rte_errno; 1030 } 1031 bits[pp / 32] |= 1 << (pp % 32); 1032 mlx5_txq_release(dev, i); 1033 } 1034 } else { 1035 for (i = 0; i < priv->rxqs_n; i++) { 1036 struct mlx5_rxq_priv *rxq = mlx5_rxq_get(dev, i); 1037 struct mlx5_rxq_ctrl *rxq_ctrl; 1038 1039 if (rxq == NULL) 1040 continue; 1041 rxq_ctrl = rxq->ctrl; 1042 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) 1043 continue; 1044 pp = rxq_ctrl->hairpin_conf.peers[0].port; 1045 if (pp >= RTE_MAX_ETHPORTS) { 1046 rte_errno = ERANGE; 1047 DRV_LOG(ERR, "port %hu queue %u peer port " 1048 "out of range %hu", 1049 priv->dev_data->port_id, i, pp); 1050 return -rte_errno; 1051 } 1052 bits[pp / 32] |= 1 << (pp % 32); 1053 } 1054 } 1055 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 1056 if (bits[i / 32] & (1 << (i % 32))) { 1057 if ((size_t)ret >= len) { 1058 rte_errno = E2BIG; 1059 return -rte_errno; 1060 } 1061 peer_ports[ret++] = i; 1062 } 1063 } 1064 return ret; 1065 } 1066 1067 /** 1068 * DPDK callback to start the device. 1069 * 1070 * Simulate device start by attaching all configured flows. 1071 * 1072 * @param dev 1073 * Pointer to Ethernet device structure. 1074 * 1075 * @return 1076 * 0 on success, a negative errno value otherwise and rte_errno is set. 1077 */ 1078 int 1079 mlx5_dev_start(struct rte_eth_dev *dev) 1080 { 1081 struct mlx5_priv *priv = dev->data->dev_private; 1082 int ret; 1083 int fine_inline; 1084 1085 DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); 1086 fine_inline = rte_mbuf_dynflag_lookup 1087 (RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, NULL); 1088 if (fine_inline >= 0) 1089 rte_net_mlx5_dynf_inline_mask = 1UL << fine_inline; 1090 else 1091 rte_net_mlx5_dynf_inline_mask = 0; 1092 if (dev->data->nb_rx_queues > 0) { 1093 ret = mlx5_dev_configure_rss_reta(dev); 1094 if (ret) { 1095 DRV_LOG(ERR, "port %u reta config failed: %s", 1096 dev->data->port_id, strerror(rte_errno)); 1097 return -rte_errno; 1098 } 1099 } 1100 ret = mlx5_txpp_start(dev); 1101 if (ret) { 1102 DRV_LOG(ERR, "port %u Tx packet pacing init failed: %s", 1103 dev->data->port_id, strerror(rte_errno)); 1104 goto error; 1105 } 1106 if ((priv->sh->devx && priv->config.dv_flow_en && 1107 priv->config.dest_tir) && priv->obj_ops.lb_dummy_queue_create) { 1108 ret = priv->obj_ops.lb_dummy_queue_create(dev); 1109 if (ret) 1110 goto error; 1111 } 1112 ret = mlx5_txq_start(dev); 1113 if (ret) { 1114 DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", 1115 dev->data->port_id, strerror(rte_errno)); 1116 goto error; 1117 } 1118 ret = mlx5_rxq_start(dev); 1119 if (ret) { 1120 DRV_LOG(ERR, "port %u Rx queue allocation failed: %s", 1121 dev->data->port_id, strerror(rte_errno)); 1122 goto error; 1123 } 1124 /* 1125 * Such step will be skipped if there is no hairpin TX queue configured 1126 * with RX peer queue from the same device. 1127 */ 1128 ret = mlx5_hairpin_auto_bind(dev); 1129 if (ret) { 1130 DRV_LOG(ERR, "port %u hairpin auto binding failed: %s", 1131 dev->data->port_id, strerror(rte_errno)); 1132 goto error; 1133 } 1134 /* Set started flag here for the following steps like control flow. */ 1135 dev->data->dev_started = 1; 1136 ret = mlx5_rx_intr_vec_enable(dev); 1137 if (ret) { 1138 DRV_LOG(ERR, "port %u Rx interrupt vector creation failed", 1139 dev->data->port_id); 1140 goto error; 1141 } 1142 mlx5_os_stats_init(dev); 1143 ret = mlx5_traffic_enable(dev); 1144 if (ret) { 1145 DRV_LOG(ERR, "port %u failed to set defaults flows", 1146 dev->data->port_id); 1147 goto error; 1148 } 1149 /* Set a mask and offset of dynamic metadata flows into Rx queues. */ 1150 mlx5_flow_rxq_dynf_metadata_set(dev); 1151 /* Set flags and context to convert Rx timestamps. */ 1152 mlx5_rxq_timestamp_set(dev); 1153 /* Set a mask and offset of scheduling on timestamp into Tx queues. */ 1154 mlx5_txq_dynf_timestamp_set(dev); 1155 /* Attach indirection table objects detached on port stop. */ 1156 ret = mlx5_action_handle_attach(dev); 1157 if (ret) { 1158 DRV_LOG(ERR, 1159 "port %u failed to attach indirect actions: %s", 1160 dev->data->port_id, rte_strerror(rte_errno)); 1161 goto error; 1162 } 1163 /* 1164 * In non-cached mode, it only needs to start the default mreg copy 1165 * action and no flow created by application exists anymore. 1166 * But it is worth wrapping the interface for further usage. 1167 */ 1168 ret = mlx5_flow_start_default(dev); 1169 if (ret) { 1170 DRV_LOG(DEBUG, "port %u failed to start default actions: %s", 1171 dev->data->port_id, strerror(rte_errno)); 1172 goto error; 1173 } 1174 if (mlx5_dev_ctx_shared_mempool_subscribe(dev) != 0) { 1175 DRV_LOG(ERR, "port %u failed to subscribe for mempool life cycle: %s", 1176 dev->data->port_id, rte_strerror(rte_errno)); 1177 goto error; 1178 } 1179 rte_wmb(); 1180 dev->tx_pkt_burst = mlx5_select_tx_function(dev); 1181 dev->rx_pkt_burst = mlx5_select_rx_function(dev); 1182 /* Enable datapath on secondary process. */ 1183 mlx5_mp_os_req_start_rxtx(dev); 1184 if (rte_intr_fd_get(priv->sh->intr_handle) >= 0) { 1185 priv->sh->port[priv->dev_port - 1].ih_port_id = 1186 (uint32_t)dev->data->port_id; 1187 } else { 1188 DRV_LOG(INFO, "port %u starts without LSC and RMV interrupts.", 1189 dev->data->port_id); 1190 dev->data->dev_conf.intr_conf.lsc = 0; 1191 dev->data->dev_conf.intr_conf.rmv = 0; 1192 } 1193 if (rte_intr_fd_get(priv->sh->intr_handle_devx) >= 0) 1194 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = 1195 (uint32_t)dev->data->port_id; 1196 return 0; 1197 error: 1198 ret = rte_errno; /* Save rte_errno before cleanup. */ 1199 /* Rollback. */ 1200 dev->data->dev_started = 0; 1201 mlx5_flow_stop_default(dev); 1202 mlx5_traffic_disable(dev); 1203 mlx5_txq_stop(dev); 1204 mlx5_rxq_stop(dev); 1205 if (priv->obj_ops.lb_dummy_queue_release) 1206 priv->obj_ops.lb_dummy_queue_release(dev); 1207 mlx5_txpp_stop(dev); /* Stop last. */ 1208 rte_errno = ret; /* Restore rte_errno. */ 1209 return -rte_errno; 1210 } 1211 1212 /** 1213 * DPDK callback to stop the device. 1214 * 1215 * Simulate device stop by detaching all configured flows. 1216 * 1217 * @param dev 1218 * Pointer to Ethernet device structure. 1219 */ 1220 int 1221 mlx5_dev_stop(struct rte_eth_dev *dev) 1222 { 1223 struct mlx5_priv *priv = dev->data->dev_private; 1224 1225 dev->data->dev_started = 0; 1226 /* Prevent crashes when queues are still in use. */ 1227 dev->rx_pkt_burst = removed_rx_burst; 1228 dev->tx_pkt_burst = removed_tx_burst; 1229 rte_wmb(); 1230 /* Disable datapath on secondary process. */ 1231 mlx5_mp_os_req_stop_rxtx(dev); 1232 rte_delay_us_sleep(1000 * priv->rxqs_n); 1233 DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); 1234 mlx5_flow_stop_default(dev); 1235 /* Control flows for default traffic can be removed firstly. */ 1236 mlx5_traffic_disable(dev); 1237 /* All RX queue flags will be cleared in the flush interface. */ 1238 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true); 1239 mlx5_flow_meter_rxq_flush(dev); 1240 mlx5_action_handle_detach(dev); 1241 mlx5_rx_intr_vec_disable(dev); 1242 priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS; 1243 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = RTE_MAX_ETHPORTS; 1244 mlx5_txq_stop(dev); 1245 mlx5_rxq_stop(dev); 1246 if (priv->obj_ops.lb_dummy_queue_release) 1247 priv->obj_ops.lb_dummy_queue_release(dev); 1248 mlx5_txpp_stop(dev); 1249 1250 return 0; 1251 } 1252 1253 /** 1254 * Enable traffic flows configured by control plane 1255 * 1256 * @param dev 1257 * Pointer to Ethernet device private data. 1258 * @param dev 1259 * Pointer to Ethernet device structure. 1260 * 1261 * @return 1262 * 0 on success, a negative errno value otherwise and rte_errno is set. 1263 */ 1264 int 1265 mlx5_traffic_enable(struct rte_eth_dev *dev) 1266 { 1267 struct mlx5_priv *priv = dev->data->dev_private; 1268 struct rte_flow_item_eth bcast = { 1269 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1270 }; 1271 struct rte_flow_item_eth ipv6_multi_spec = { 1272 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", 1273 }; 1274 struct rte_flow_item_eth ipv6_multi_mask = { 1275 .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00", 1276 }; 1277 struct rte_flow_item_eth unicast = { 1278 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1279 }; 1280 struct rte_flow_item_eth unicast_mask = { 1281 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 1282 }; 1283 const unsigned int vlan_filter_n = priv->vlan_filter_n; 1284 const struct rte_ether_addr cmp = { 1285 .addr_bytes = "\x00\x00\x00\x00\x00\x00", 1286 }; 1287 unsigned int i; 1288 unsigned int j; 1289 int ret; 1290 1291 /* 1292 * Hairpin txq default flow should be created no matter if it is 1293 * isolation mode. Or else all the packets to be sent will be sent 1294 * out directly without the TX flow actions, e.g. encapsulation. 1295 */ 1296 for (i = 0; i != priv->txqs_n; ++i) { 1297 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 1298 if (!txq_ctrl) 1299 continue; 1300 /* Only Tx implicit mode requires the default Tx flow. */ 1301 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN && 1302 txq_ctrl->hairpin_conf.tx_explicit == 0 && 1303 txq_ctrl->hairpin_conf.peers[0].port == 1304 priv->dev_data->port_id) { 1305 ret = mlx5_ctrl_flow_source_queue(dev, i); 1306 if (ret) { 1307 mlx5_txq_release(dev, i); 1308 goto error; 1309 } 1310 } 1311 if ((priv->representor || priv->master) && 1312 priv->config.dv_esw_en) { 1313 if (mlx5_flow_create_devx_sq_miss_flow(dev, i) == 0) { 1314 DRV_LOG(ERR, 1315 "Port %u Tx queue %u SQ create representor devx default miss rule failed.", 1316 dev->data->port_id, i); 1317 goto error; 1318 } 1319 } 1320 mlx5_txq_release(dev, i); 1321 } 1322 if ((priv->master || priv->representor) && priv->config.dv_esw_en) { 1323 if (mlx5_flow_create_esw_table_zero_flow(dev)) 1324 priv->fdb_def_rule = 1; 1325 else 1326 DRV_LOG(INFO, "port %u FDB default rule cannot be" 1327 " configured - only Eswitch group 0 flows are" 1328 " supported.", dev->data->port_id); 1329 } 1330 if (!priv->config.lacp_by_user && priv->pf_bond >= 0) { 1331 ret = mlx5_flow_lacp_miss(dev); 1332 if (ret) 1333 DRV_LOG(INFO, "port %u LACP rule cannot be created - " 1334 "forward LACP to kernel.", dev->data->port_id); 1335 else 1336 DRV_LOG(INFO, "LACP traffic will be missed in port %u." 1337 , dev->data->port_id); 1338 } 1339 if (priv->isolated) 1340 return 0; 1341 if (dev->data->promiscuous) { 1342 struct rte_flow_item_eth promisc = { 1343 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1344 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1345 .type = 0, 1346 }; 1347 1348 ret = mlx5_ctrl_flow(dev, &promisc, &promisc); 1349 if (ret) 1350 goto error; 1351 } 1352 if (dev->data->all_multicast) { 1353 struct rte_flow_item_eth multicast = { 1354 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", 1355 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 1356 .type = 0, 1357 }; 1358 1359 ret = mlx5_ctrl_flow(dev, &multicast, &multicast); 1360 if (ret) 1361 goto error; 1362 } else { 1363 /* Add broadcast/multicast flows. */ 1364 for (i = 0; i != vlan_filter_n; ++i) { 1365 uint16_t vlan = priv->vlan_filter[i]; 1366 1367 struct rte_flow_item_vlan vlan_spec = { 1368 .tci = rte_cpu_to_be_16(vlan), 1369 }; 1370 struct rte_flow_item_vlan vlan_mask = 1371 rte_flow_item_vlan_mask; 1372 1373 ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, 1374 &vlan_spec, &vlan_mask); 1375 if (ret) 1376 goto error; 1377 ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec, 1378 &ipv6_multi_mask, 1379 &vlan_spec, &vlan_mask); 1380 if (ret) 1381 goto error; 1382 } 1383 if (!vlan_filter_n) { 1384 ret = mlx5_ctrl_flow(dev, &bcast, &bcast); 1385 if (ret) 1386 goto error; 1387 ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec, 1388 &ipv6_multi_mask); 1389 if (ret) { 1390 /* Do not fail on IPv6 broadcast creation failure. */ 1391 DRV_LOG(WARNING, 1392 "IPv6 broadcast is not supported"); 1393 ret = 0; 1394 } 1395 } 1396 } 1397 /* Add MAC address flows. */ 1398 for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) { 1399 struct rte_ether_addr *mac = &dev->data->mac_addrs[i]; 1400 1401 if (!memcmp(mac, &cmp, sizeof(*mac))) 1402 continue; 1403 memcpy(&unicast.dst.addr_bytes, 1404 mac->addr_bytes, 1405 RTE_ETHER_ADDR_LEN); 1406 for (j = 0; j != vlan_filter_n; ++j) { 1407 uint16_t vlan = priv->vlan_filter[j]; 1408 1409 struct rte_flow_item_vlan vlan_spec = { 1410 .tci = rte_cpu_to_be_16(vlan), 1411 }; 1412 struct rte_flow_item_vlan vlan_mask = 1413 rte_flow_item_vlan_mask; 1414 1415 ret = mlx5_ctrl_flow_vlan(dev, &unicast, 1416 &unicast_mask, 1417 &vlan_spec, 1418 &vlan_mask); 1419 if (ret) 1420 goto error; 1421 } 1422 if (!vlan_filter_n) { 1423 ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask); 1424 if (ret) 1425 goto error; 1426 } 1427 } 1428 return 0; 1429 error: 1430 ret = rte_errno; /* Save rte_errno before cleanup. */ 1431 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false); 1432 rte_errno = ret; /* Restore rte_errno. */ 1433 return -rte_errno; 1434 } 1435 1436 1437 /** 1438 * Disable traffic flows configured by control plane 1439 * 1440 * @param dev 1441 * Pointer to Ethernet device private data. 1442 */ 1443 void 1444 mlx5_traffic_disable(struct rte_eth_dev *dev) 1445 { 1446 mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false); 1447 } 1448 1449 /** 1450 * Restart traffic flows configured by control plane 1451 * 1452 * @param dev 1453 * Pointer to Ethernet device private data. 1454 * 1455 * @return 1456 * 0 on success, a negative errno value otherwise and rte_errno is set. 1457 */ 1458 int 1459 mlx5_traffic_restart(struct rte_eth_dev *dev) 1460 { 1461 if (dev->data->dev_started) { 1462 mlx5_traffic_disable(dev); 1463 return mlx5_traffic_enable(dev); 1464 } 1465 return 0; 1466 } 1467