1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <unistd.h> 7 8 #include <rte_ether.h> 9 #include <rte_ethdev_driver.h> 10 #include <rte_interrupts.h> 11 #include <rte_alarm.h> 12 13 #include "mlx5.h" 14 #include "mlx5_mr.h" 15 #include "mlx5_rxtx.h" 16 #include "mlx5_utils.h" 17 #include "rte_pmd_mlx5.h" 18 19 /** 20 * Stop traffic on Tx queues. 21 * 22 * @param dev 23 * Pointer to Ethernet device structure. 24 */ 25 static void 26 mlx5_txq_stop(struct rte_eth_dev *dev) 27 { 28 struct mlx5_priv *priv = dev->data->dev_private; 29 unsigned int i; 30 31 for (i = 0; i != priv->txqs_n; ++i) 32 mlx5_txq_release(dev, i); 33 } 34 35 /** 36 * Start traffic on Tx queues. 37 * 38 * @param dev 39 * Pointer to Ethernet device structure. 40 * 41 * @return 42 * 0 on success, a negative errno value otherwise and rte_errno is set. 43 */ 44 static int 45 mlx5_txq_start(struct rte_eth_dev *dev) 46 { 47 struct mlx5_priv *priv = dev->data->dev_private; 48 unsigned int i; 49 int ret; 50 51 for (i = 0; i != priv->txqs_n; ++i) { 52 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 53 54 if (!txq_ctrl) 55 continue; 56 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 57 txq_ctrl->obj = mlx5_txq_obj_new 58 (dev, i, MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN); 59 } else { 60 txq_alloc_elts(txq_ctrl); 61 txq_ctrl->obj = mlx5_txq_obj_new 62 (dev, i, priv->txpp_en ? 63 MLX5_TXQ_OBJ_TYPE_DEVX_SQ : 64 MLX5_TXQ_OBJ_TYPE_IBV); 65 } 66 if (!txq_ctrl->obj) { 67 rte_errno = ENOMEM; 68 goto error; 69 } 70 } 71 return 0; 72 error: 73 ret = rte_errno; /* Save rte_errno before cleanup. */ 74 do { 75 mlx5_txq_release(dev, i); 76 } while (i-- != 0); 77 rte_errno = ret; /* Restore rte_errno. */ 78 return -rte_errno; 79 } 80 81 /** 82 * Stop traffic on Rx queues. 83 * 84 * @param dev 85 * Pointer to Ethernet device structure. 86 */ 87 static void 88 mlx5_rxq_stop(struct rte_eth_dev *dev) 89 { 90 struct mlx5_priv *priv = dev->data->dev_private; 91 unsigned int i; 92 93 for (i = 0; i != priv->rxqs_n; ++i) 94 mlx5_rxq_release(dev, i); 95 } 96 97 /** 98 * Start traffic on Rx queues. 99 * 100 * @param dev 101 * Pointer to Ethernet device structure. 102 * 103 * @return 104 * 0 on success, a negative errno value otherwise and rte_errno is set. 105 */ 106 static int 107 mlx5_rxq_start(struct rte_eth_dev *dev) 108 { 109 struct mlx5_priv *priv = dev->data->dev_private; 110 unsigned int i; 111 int ret = 0; 112 enum mlx5_rxq_obj_type obj_type = MLX5_RXQ_OBJ_TYPE_IBV; 113 struct mlx5_rxq_data *rxq = NULL; 114 115 for (i = 0; i < priv->rxqs_n; ++i) { 116 rxq = (*priv->rxqs)[i]; 117 if (rxq && rxq->lro) { 118 obj_type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ; 119 break; 120 } 121 } 122 /* Allocate/reuse/resize mempool for Multi-Packet RQ. */ 123 if (mlx5_mprq_alloc_mp(dev)) { 124 /* Should not release Rx queues but return immediately. */ 125 return -rte_errno; 126 } 127 for (i = 0; i != priv->rxqs_n; ++i) { 128 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i); 129 struct rte_mempool *mp; 130 131 if (!rxq_ctrl) 132 continue; 133 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) { 134 rxq_ctrl->obj = mlx5_rxq_obj_new 135 (dev, i, MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN); 136 if (!rxq_ctrl->obj) 137 goto error; 138 continue; 139 } 140 /* Pre-register Rx mempool. */ 141 mp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 142 rxq_ctrl->rxq.mprq_mp : rxq_ctrl->rxq.mp; 143 DRV_LOG(DEBUG, 144 "port %u Rx queue %u registering" 145 " mp %s having %u chunks", 146 dev->data->port_id, rxq_ctrl->rxq.idx, 147 mp->name, mp->nb_mem_chunks); 148 mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp); 149 ret = rxq_alloc_elts(rxq_ctrl); 150 if (ret) 151 goto error; 152 rxq_ctrl->obj = mlx5_rxq_obj_new(dev, i, obj_type); 153 if (!rxq_ctrl->obj) 154 goto error; 155 if (obj_type == MLX5_RXQ_OBJ_TYPE_IBV) 156 rxq_ctrl->wqn = rxq_ctrl->obj->wq->wq_num; 157 else if (obj_type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) 158 rxq_ctrl->wqn = rxq_ctrl->obj->rq->id; 159 } 160 return 0; 161 error: 162 ret = rte_errno; /* Save rte_errno before cleanup. */ 163 do { 164 mlx5_rxq_release(dev, i); 165 } while (i-- != 0); 166 rte_errno = ret; /* Restore rte_errno. */ 167 return -rte_errno; 168 } 169 170 /** 171 * Binds Tx queues to Rx queues for hairpin. 172 * 173 * Binds Tx queues to the target Rx queues. 174 * 175 * @param dev 176 * Pointer to Ethernet device structure. 177 * 178 * @return 179 * 0 on success, a negative errno value otherwise and rte_errno is set. 180 */ 181 static int 182 mlx5_hairpin_bind(struct rte_eth_dev *dev) 183 { 184 struct mlx5_priv *priv = dev->data->dev_private; 185 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 186 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 187 struct mlx5_txq_ctrl *txq_ctrl; 188 struct mlx5_rxq_ctrl *rxq_ctrl; 189 struct mlx5_devx_obj *sq; 190 struct mlx5_devx_obj *rq; 191 unsigned int i; 192 int ret = 0; 193 194 for (i = 0; i != priv->txqs_n; ++i) { 195 txq_ctrl = mlx5_txq_get(dev, i); 196 if (!txq_ctrl) 197 continue; 198 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 199 mlx5_txq_release(dev, i); 200 continue; 201 } 202 if (!txq_ctrl->obj) { 203 rte_errno = ENOMEM; 204 DRV_LOG(ERR, "port %u no txq object found: %d", 205 dev->data->port_id, i); 206 mlx5_txq_release(dev, i); 207 return -rte_errno; 208 } 209 sq = txq_ctrl->obj->sq; 210 rxq_ctrl = mlx5_rxq_get(dev, 211 txq_ctrl->hairpin_conf.peers[0].queue); 212 if (!rxq_ctrl) { 213 mlx5_txq_release(dev, i); 214 rte_errno = EINVAL; 215 DRV_LOG(ERR, "port %u no rxq object found: %d", 216 dev->data->port_id, 217 txq_ctrl->hairpin_conf.peers[0].queue); 218 return -rte_errno; 219 } 220 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN || 221 rxq_ctrl->hairpin_conf.peers[0].queue != i) { 222 rte_errno = ENOMEM; 223 DRV_LOG(ERR, "port %u Tx queue %d can't be binded to " 224 "Rx queue %d", dev->data->port_id, 225 i, txq_ctrl->hairpin_conf.peers[0].queue); 226 goto error; 227 } 228 rq = rxq_ctrl->obj->rq; 229 if (!rq) { 230 rte_errno = ENOMEM; 231 DRV_LOG(ERR, "port %u hairpin no matching rxq: %d", 232 dev->data->port_id, 233 txq_ctrl->hairpin_conf.peers[0].queue); 234 goto error; 235 } 236 sq_attr.state = MLX5_SQC_STATE_RDY; 237 sq_attr.sq_state = MLX5_SQC_STATE_RST; 238 sq_attr.hairpin_peer_rq = rq->id; 239 sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 240 ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr); 241 if (ret) 242 goto error; 243 rq_attr.state = MLX5_SQC_STATE_RDY; 244 rq_attr.rq_state = MLX5_SQC_STATE_RST; 245 rq_attr.hairpin_peer_sq = sq->id; 246 rq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 247 ret = mlx5_devx_cmd_modify_rq(rq, &rq_attr); 248 if (ret) 249 goto error; 250 mlx5_txq_release(dev, i); 251 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 252 } 253 return 0; 254 error: 255 mlx5_txq_release(dev, i); 256 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 257 return -rte_errno; 258 } 259 260 /** 261 * DPDK callback to start the device. 262 * 263 * Simulate device start by attaching all configured flows. 264 * 265 * @param dev 266 * Pointer to Ethernet device structure. 267 * 268 * @return 269 * 0 on success, a negative errno value otherwise and rte_errno is set. 270 */ 271 int 272 mlx5_dev_start(struct rte_eth_dev *dev) 273 { 274 struct mlx5_priv *priv = dev->data->dev_private; 275 int ret; 276 int fine_inline; 277 278 DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); 279 fine_inline = rte_mbuf_dynflag_lookup 280 (RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, NULL); 281 if (fine_inline > 0) 282 rte_net_mlx5_dynf_inline_mask = 1UL << fine_inline; 283 else 284 rte_net_mlx5_dynf_inline_mask = 0; 285 if (dev->data->nb_rx_queues > 0) { 286 ret = mlx5_dev_configure_rss_reta(dev); 287 if (ret) { 288 DRV_LOG(ERR, "port %u reta config failed: %s", 289 dev->data->port_id, strerror(rte_errno)); 290 return -rte_errno; 291 } 292 } 293 ret = mlx5_txpp_start(dev); 294 if (ret) { 295 DRV_LOG(ERR, "port %u Tx packet pacing init failed: %s", 296 dev->data->port_id, strerror(rte_errno)); 297 goto error; 298 } 299 ret = mlx5_txq_start(dev); 300 if (ret) { 301 DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", 302 dev->data->port_id, strerror(rte_errno)); 303 goto error; 304 } 305 ret = mlx5_rxq_start(dev); 306 if (ret) { 307 DRV_LOG(ERR, "port %u Rx queue allocation failed: %s", 308 dev->data->port_id, strerror(rte_errno)); 309 goto error; 310 } 311 ret = mlx5_hairpin_bind(dev); 312 if (ret) { 313 DRV_LOG(ERR, "port %u hairpin binding failed: %s", 314 dev->data->port_id, strerror(rte_errno)); 315 goto error; 316 } 317 /* Set started flag here for the following steps like control flow. */ 318 dev->data->dev_started = 1; 319 ret = mlx5_rx_intr_vec_enable(dev); 320 if (ret) { 321 DRV_LOG(ERR, "port %u Rx interrupt vector creation failed", 322 dev->data->port_id); 323 goto error; 324 } 325 mlx5_os_stats_init(dev); 326 ret = mlx5_traffic_enable(dev); 327 if (ret) { 328 DRV_LOG(ERR, "port %u failed to set defaults flows", 329 dev->data->port_id); 330 goto error; 331 } 332 /* Set a mask and offset of dynamic metadata flows into Rx queues*/ 333 mlx5_flow_rxq_dynf_metadata_set(dev); 334 /* 335 * In non-cached mode, it only needs to start the default mreg copy 336 * action and no flow created by application exists anymore. 337 * But it is worth wrapping the interface for further usage. 338 */ 339 ret = mlx5_flow_start_default(dev); 340 if (ret) { 341 DRV_LOG(DEBUG, "port %u failed to start default actions: %s", 342 dev->data->port_id, strerror(rte_errno)); 343 goto error; 344 } 345 rte_wmb(); 346 dev->tx_pkt_burst = mlx5_select_tx_function(dev); 347 dev->rx_pkt_burst = mlx5_select_rx_function(dev); 348 /* Enable datapath on secondary process. */ 349 mlx5_mp_req_start_rxtx(dev); 350 if (priv->sh->intr_handle.fd >= 0) { 351 priv->sh->port[priv->dev_port - 1].ih_port_id = 352 (uint32_t)dev->data->port_id; 353 } else { 354 DRV_LOG(INFO, "port %u starts without LSC and RMV interrupts.", 355 dev->data->port_id); 356 dev->data->dev_conf.intr_conf.lsc = 0; 357 dev->data->dev_conf.intr_conf.rmv = 0; 358 } 359 if (priv->sh->intr_handle_devx.fd >= 0) 360 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = 361 (uint32_t)dev->data->port_id; 362 return 0; 363 error: 364 ret = rte_errno; /* Save rte_errno before cleanup. */ 365 /* Rollback. */ 366 dev->data->dev_started = 0; 367 mlx5_flow_stop_default(dev); 368 mlx5_traffic_disable(dev); 369 mlx5_txq_stop(dev); 370 mlx5_rxq_stop(dev); 371 mlx5_txpp_stop(dev); /* Stop last. */ 372 rte_errno = ret; /* Restore rte_errno. */ 373 return -rte_errno; 374 } 375 376 /** 377 * DPDK callback to stop the device. 378 * 379 * Simulate device stop by detaching all configured flows. 380 * 381 * @param dev 382 * Pointer to Ethernet device structure. 383 */ 384 void 385 mlx5_dev_stop(struct rte_eth_dev *dev) 386 { 387 struct mlx5_priv *priv = dev->data->dev_private; 388 389 dev->data->dev_started = 0; 390 /* Prevent crashes when queues are still in use. */ 391 dev->rx_pkt_burst = removed_rx_burst; 392 dev->tx_pkt_burst = removed_tx_burst; 393 rte_wmb(); 394 /* Disable datapath on secondary process. */ 395 mlx5_mp_req_stop_rxtx(dev); 396 usleep(1000 * priv->rxqs_n); 397 DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); 398 mlx5_flow_stop_default(dev); 399 /* Control flows for default traffic can be removed firstly. */ 400 mlx5_traffic_disable(dev); 401 /* All RX queue flags will be cleared in the flush interface. */ 402 mlx5_flow_list_flush(dev, &priv->flows, true); 403 mlx5_rx_intr_vec_disable(dev); 404 priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS; 405 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = RTE_MAX_ETHPORTS; 406 mlx5_txq_stop(dev); 407 mlx5_rxq_stop(dev); 408 mlx5_txpp_stop(dev); 409 } 410 411 /** 412 * Enable traffic flows configured by control plane 413 * 414 * @param dev 415 * Pointer to Ethernet device private data. 416 * @param dev 417 * Pointer to Ethernet device structure. 418 * 419 * @return 420 * 0 on success, a negative errno value otherwise and rte_errno is set. 421 */ 422 int 423 mlx5_traffic_enable(struct rte_eth_dev *dev) 424 { 425 struct mlx5_priv *priv = dev->data->dev_private; 426 struct rte_flow_item_eth bcast = { 427 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 428 }; 429 struct rte_flow_item_eth ipv6_multi_spec = { 430 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", 431 }; 432 struct rte_flow_item_eth ipv6_multi_mask = { 433 .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00", 434 }; 435 struct rte_flow_item_eth unicast = { 436 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 437 }; 438 struct rte_flow_item_eth unicast_mask = { 439 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 440 }; 441 const unsigned int vlan_filter_n = priv->vlan_filter_n; 442 const struct rte_ether_addr cmp = { 443 .addr_bytes = "\x00\x00\x00\x00\x00\x00", 444 }; 445 unsigned int i; 446 unsigned int j; 447 int ret; 448 449 /* 450 * Hairpin txq default flow should be created no matter if it is 451 * isolation mode. Or else all the packets to be sent will be sent 452 * out directly without the TX flow actions, e.g. encapsulation. 453 */ 454 for (i = 0; i != priv->txqs_n; ++i) { 455 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 456 if (!txq_ctrl) 457 continue; 458 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 459 ret = mlx5_ctrl_flow_source_queue(dev, i); 460 if (ret) { 461 mlx5_txq_release(dev, i); 462 goto error; 463 } 464 } 465 mlx5_txq_release(dev, i); 466 } 467 if (priv->config.dv_esw_en && !priv->config.vf) { 468 if (mlx5_flow_create_esw_table_zero_flow(dev)) 469 priv->fdb_def_rule = 1; 470 else 471 DRV_LOG(INFO, "port %u FDB default rule cannot be" 472 " configured - only Eswitch group 0 flows are" 473 " supported.", dev->data->port_id); 474 } 475 if (!priv->config.lacp_by_user && priv->pf_bond >= 0) { 476 ret = mlx5_flow_lacp_miss(dev); 477 if (ret) 478 DRV_LOG(INFO, "port %u LACP rule cannot be created - " 479 "forward LACP to kernel.", dev->data->port_id); 480 else 481 DRV_LOG(INFO, "LACP traffic will be missed in port %u." 482 , dev->data->port_id); 483 } 484 if (priv->isolated) 485 return 0; 486 if (dev->data->promiscuous) { 487 struct rte_flow_item_eth promisc = { 488 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 489 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 490 .type = 0, 491 }; 492 493 ret = mlx5_ctrl_flow(dev, &promisc, &promisc); 494 if (ret) 495 goto error; 496 } 497 if (dev->data->all_multicast) { 498 struct rte_flow_item_eth multicast = { 499 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", 500 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 501 .type = 0, 502 }; 503 504 ret = mlx5_ctrl_flow(dev, &multicast, &multicast); 505 if (ret) 506 goto error; 507 } else { 508 /* Add broadcast/multicast flows. */ 509 for (i = 0; i != vlan_filter_n; ++i) { 510 uint16_t vlan = priv->vlan_filter[i]; 511 512 struct rte_flow_item_vlan vlan_spec = { 513 .tci = rte_cpu_to_be_16(vlan), 514 }; 515 struct rte_flow_item_vlan vlan_mask = 516 rte_flow_item_vlan_mask; 517 518 ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, 519 &vlan_spec, &vlan_mask); 520 if (ret) 521 goto error; 522 ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec, 523 &ipv6_multi_mask, 524 &vlan_spec, &vlan_mask); 525 if (ret) 526 goto error; 527 } 528 if (!vlan_filter_n) { 529 ret = mlx5_ctrl_flow(dev, &bcast, &bcast); 530 if (ret) 531 goto error; 532 ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec, 533 &ipv6_multi_mask); 534 if (ret) 535 goto error; 536 } 537 } 538 /* Add MAC address flows. */ 539 for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) { 540 struct rte_ether_addr *mac = &dev->data->mac_addrs[i]; 541 542 if (!memcmp(mac, &cmp, sizeof(*mac))) 543 continue; 544 memcpy(&unicast.dst.addr_bytes, 545 mac->addr_bytes, 546 RTE_ETHER_ADDR_LEN); 547 for (j = 0; j != vlan_filter_n; ++j) { 548 uint16_t vlan = priv->vlan_filter[j]; 549 550 struct rte_flow_item_vlan vlan_spec = { 551 .tci = rte_cpu_to_be_16(vlan), 552 }; 553 struct rte_flow_item_vlan vlan_mask = 554 rte_flow_item_vlan_mask; 555 556 ret = mlx5_ctrl_flow_vlan(dev, &unicast, 557 &unicast_mask, 558 &vlan_spec, 559 &vlan_mask); 560 if (ret) 561 goto error; 562 } 563 if (!vlan_filter_n) { 564 ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask); 565 if (ret) 566 goto error; 567 } 568 } 569 return 0; 570 error: 571 ret = rte_errno; /* Save rte_errno before cleanup. */ 572 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false); 573 rte_errno = ret; /* Restore rte_errno. */ 574 return -rte_errno; 575 } 576 577 578 /** 579 * Disable traffic flows configured by control plane 580 * 581 * @param dev 582 * Pointer to Ethernet device private data. 583 */ 584 void 585 mlx5_traffic_disable(struct rte_eth_dev *dev) 586 { 587 struct mlx5_priv *priv = dev->data->dev_private; 588 589 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false); 590 } 591 592 /** 593 * Restart traffic flows configured by control plane 594 * 595 * @param dev 596 * Pointer to Ethernet device private data. 597 * 598 * @return 599 * 0 on success, a negative errno value otherwise and rte_errno is set. 600 */ 601 int 602 mlx5_traffic_restart(struct rte_eth_dev *dev) 603 { 604 if (dev->data->dev_started) { 605 mlx5_traffic_disable(dev); 606 return mlx5_traffic_enable(dev); 607 } 608 return 0; 609 } 610