1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <unistd.h> 7 8 #include <rte_ether.h> 9 #include <rte_ethdev_driver.h> 10 #include <rte_interrupts.h> 11 #include <rte_alarm.h> 12 13 #include "mlx5.h" 14 #include "mlx5_mr.h" 15 #include "mlx5_rxtx.h" 16 #include "mlx5_utils.h" 17 #include "rte_pmd_mlx5.h" 18 19 /** 20 * Stop traffic on Tx queues. 21 * 22 * @param dev 23 * Pointer to Ethernet device structure. 24 */ 25 static void 26 mlx5_txq_stop(struct rte_eth_dev *dev) 27 { 28 struct mlx5_priv *priv = dev->data->dev_private; 29 unsigned int i; 30 31 for (i = 0; i != priv->txqs_n; ++i) 32 mlx5_txq_release(dev, i); 33 } 34 35 /** 36 * Start traffic on Tx queues. 37 * 38 * @param dev 39 * Pointer to Ethernet device structure. 40 * 41 * @return 42 * 0 on success, a negative errno value otherwise and rte_errno is set. 43 */ 44 static int 45 mlx5_txq_start(struct rte_eth_dev *dev) 46 { 47 struct mlx5_priv *priv = dev->data->dev_private; 48 unsigned int i; 49 int ret; 50 51 for (i = 0; i != priv->txqs_n; ++i) { 52 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 53 54 if (!txq_ctrl) 55 continue; 56 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 57 txq_ctrl->obj = mlx5_txq_obj_new 58 (dev, i, MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN); 59 } else { 60 txq_alloc_elts(txq_ctrl); 61 txq_ctrl->obj = mlx5_txq_obj_new 62 (dev, i, MLX5_TXQ_OBJ_TYPE_IBV); 63 } 64 if (!txq_ctrl->obj) { 65 rte_errno = ENOMEM; 66 goto error; 67 } 68 } 69 return 0; 70 error: 71 ret = rte_errno; /* Save rte_errno before cleanup. */ 72 do { 73 mlx5_txq_release(dev, i); 74 } while (i-- != 0); 75 rte_errno = ret; /* Restore rte_errno. */ 76 return -rte_errno; 77 } 78 79 /** 80 * Stop traffic on Rx queues. 81 * 82 * @param dev 83 * Pointer to Ethernet device structure. 84 */ 85 static void 86 mlx5_rxq_stop(struct rte_eth_dev *dev) 87 { 88 struct mlx5_priv *priv = dev->data->dev_private; 89 unsigned int i; 90 91 for (i = 0; i != priv->rxqs_n; ++i) 92 mlx5_rxq_release(dev, i); 93 } 94 95 /** 96 * Start traffic on Rx queues. 97 * 98 * @param dev 99 * Pointer to Ethernet device structure. 100 * 101 * @return 102 * 0 on success, a negative errno value otherwise and rte_errno is set. 103 */ 104 static int 105 mlx5_rxq_start(struct rte_eth_dev *dev) 106 { 107 struct mlx5_priv *priv = dev->data->dev_private; 108 unsigned int i; 109 int ret = 0; 110 enum mlx5_rxq_obj_type obj_type = MLX5_RXQ_OBJ_TYPE_IBV; 111 struct mlx5_rxq_data *rxq = NULL; 112 113 for (i = 0; i < priv->rxqs_n; ++i) { 114 rxq = (*priv->rxqs)[i]; 115 if (rxq && rxq->lro) { 116 obj_type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ; 117 break; 118 } 119 } 120 /* Allocate/reuse/resize mempool for Multi-Packet RQ. */ 121 if (mlx5_mprq_alloc_mp(dev)) { 122 /* Should not release Rx queues but return immediately. */ 123 return -rte_errno; 124 } 125 for (i = 0; i != priv->rxqs_n; ++i) { 126 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i); 127 struct rte_mempool *mp; 128 129 if (!rxq_ctrl) 130 continue; 131 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) { 132 rxq_ctrl->obj = mlx5_rxq_obj_new 133 (dev, i, MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN); 134 if (!rxq_ctrl->obj) 135 goto error; 136 continue; 137 } 138 /* Pre-register Rx mempool. */ 139 mp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 140 rxq_ctrl->rxq.mprq_mp : rxq_ctrl->rxq.mp; 141 DRV_LOG(DEBUG, 142 "port %u Rx queue %u registering" 143 " mp %s having %u chunks", 144 dev->data->port_id, rxq_ctrl->rxq.idx, 145 mp->name, mp->nb_mem_chunks); 146 mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp); 147 ret = rxq_alloc_elts(rxq_ctrl); 148 if (ret) 149 goto error; 150 rxq_ctrl->obj = mlx5_rxq_obj_new(dev, i, obj_type); 151 if (!rxq_ctrl->obj) 152 goto error; 153 if (obj_type == MLX5_RXQ_OBJ_TYPE_IBV) 154 rxq_ctrl->wqn = rxq_ctrl->obj->wq->wq_num; 155 else if (obj_type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) 156 rxq_ctrl->wqn = rxq_ctrl->obj->rq->id; 157 } 158 return 0; 159 error: 160 ret = rte_errno; /* Save rte_errno before cleanup. */ 161 do { 162 mlx5_rxq_release(dev, i); 163 } while (i-- != 0); 164 rte_errno = ret; /* Restore rte_errno. */ 165 return -rte_errno; 166 } 167 168 /** 169 * Binds Tx queues to Rx queues for hairpin. 170 * 171 * Binds Tx queues to the target Rx queues. 172 * 173 * @param dev 174 * Pointer to Ethernet device structure. 175 * 176 * @return 177 * 0 on success, a negative errno value otherwise and rte_errno is set. 178 */ 179 static int 180 mlx5_hairpin_bind(struct rte_eth_dev *dev) 181 { 182 struct mlx5_priv *priv = dev->data->dev_private; 183 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 184 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 185 struct mlx5_txq_ctrl *txq_ctrl; 186 struct mlx5_rxq_ctrl *rxq_ctrl; 187 struct mlx5_devx_obj *sq; 188 struct mlx5_devx_obj *rq; 189 unsigned int i; 190 int ret = 0; 191 192 for (i = 0; i != priv->txqs_n; ++i) { 193 txq_ctrl = mlx5_txq_get(dev, i); 194 if (!txq_ctrl) 195 continue; 196 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 197 mlx5_txq_release(dev, i); 198 continue; 199 } 200 if (!txq_ctrl->obj) { 201 rte_errno = ENOMEM; 202 DRV_LOG(ERR, "port %u no txq object found: %d", 203 dev->data->port_id, i); 204 mlx5_txq_release(dev, i); 205 return -rte_errno; 206 } 207 sq = txq_ctrl->obj->sq; 208 rxq_ctrl = mlx5_rxq_get(dev, 209 txq_ctrl->hairpin_conf.peers[0].queue); 210 if (!rxq_ctrl) { 211 mlx5_txq_release(dev, i); 212 rte_errno = EINVAL; 213 DRV_LOG(ERR, "port %u no rxq object found: %d", 214 dev->data->port_id, 215 txq_ctrl->hairpin_conf.peers[0].queue); 216 return -rte_errno; 217 } 218 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN || 219 rxq_ctrl->hairpin_conf.peers[0].queue != i) { 220 rte_errno = ENOMEM; 221 DRV_LOG(ERR, "port %u Tx queue %d can't be binded to " 222 "Rx queue %d", dev->data->port_id, 223 i, txq_ctrl->hairpin_conf.peers[0].queue); 224 goto error; 225 } 226 rq = rxq_ctrl->obj->rq; 227 if (!rq) { 228 rte_errno = ENOMEM; 229 DRV_LOG(ERR, "port %u hairpin no matching rxq: %d", 230 dev->data->port_id, 231 txq_ctrl->hairpin_conf.peers[0].queue); 232 goto error; 233 } 234 sq_attr.state = MLX5_SQC_STATE_RDY; 235 sq_attr.sq_state = MLX5_SQC_STATE_RST; 236 sq_attr.hairpin_peer_rq = rq->id; 237 sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 238 ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr); 239 if (ret) 240 goto error; 241 rq_attr.state = MLX5_SQC_STATE_RDY; 242 rq_attr.rq_state = MLX5_SQC_STATE_RST; 243 rq_attr.hairpin_peer_sq = sq->id; 244 rq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 245 ret = mlx5_devx_cmd_modify_rq(rq, &rq_attr); 246 if (ret) 247 goto error; 248 mlx5_txq_release(dev, i); 249 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 250 } 251 return 0; 252 error: 253 mlx5_txq_release(dev, i); 254 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 255 return -rte_errno; 256 } 257 258 /** 259 * DPDK callback to start the device. 260 * 261 * Simulate device start by attaching all configured flows. 262 * 263 * @param dev 264 * Pointer to Ethernet device structure. 265 * 266 * @return 267 * 0 on success, a negative errno value otherwise and rte_errno is set. 268 */ 269 int 270 mlx5_dev_start(struct rte_eth_dev *dev) 271 { 272 struct mlx5_priv *priv = dev->data->dev_private; 273 int ret; 274 int fine_inline; 275 276 DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); 277 fine_inline = rte_mbuf_dynflag_lookup 278 (RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, NULL); 279 if (fine_inline > 0) 280 rte_net_mlx5_dynf_inline_mask = 1UL << fine_inline; 281 else 282 rte_net_mlx5_dynf_inline_mask = 0; 283 if (dev->data->nb_rx_queues > 0) { 284 ret = mlx5_dev_configure_rss_reta(dev); 285 if (ret) { 286 DRV_LOG(ERR, "port %u reta config failed: %s", 287 dev->data->port_id, strerror(rte_errno)); 288 return -rte_errno; 289 } 290 } 291 ret = mlx5_txpp_start(dev); 292 if (ret) { 293 DRV_LOG(ERR, "port %u Tx packet pacing init failed: %s", 294 dev->data->port_id, strerror(rte_errno)); 295 goto error; 296 } 297 ret = mlx5_txq_start(dev); 298 if (ret) { 299 DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", 300 dev->data->port_id, strerror(rte_errno)); 301 goto error; 302 } 303 ret = mlx5_rxq_start(dev); 304 if (ret) { 305 DRV_LOG(ERR, "port %u Rx queue allocation failed: %s", 306 dev->data->port_id, strerror(rte_errno)); 307 goto error; 308 } 309 ret = mlx5_hairpin_bind(dev); 310 if (ret) { 311 DRV_LOG(ERR, "port %u hairpin binding failed: %s", 312 dev->data->port_id, strerror(rte_errno)); 313 goto error; 314 } 315 /* Set started flag here for the following steps like control flow. */ 316 dev->data->dev_started = 1; 317 ret = mlx5_rx_intr_vec_enable(dev); 318 if (ret) { 319 DRV_LOG(ERR, "port %u Rx interrupt vector creation failed", 320 dev->data->port_id); 321 goto error; 322 } 323 mlx5_os_stats_init(dev); 324 ret = mlx5_traffic_enable(dev); 325 if (ret) { 326 DRV_LOG(ERR, "port %u failed to set defaults flows", 327 dev->data->port_id); 328 goto error; 329 } 330 /* Set a mask and offset of dynamic metadata flows into Rx queues*/ 331 mlx5_flow_rxq_dynf_metadata_set(dev); 332 /* 333 * In non-cached mode, it only needs to start the default mreg copy 334 * action and no flow created by application exists anymore. 335 * But it is worth wrapping the interface for further usage. 336 */ 337 ret = mlx5_flow_start_default(dev); 338 if (ret) { 339 DRV_LOG(DEBUG, "port %u failed to start default actions: %s", 340 dev->data->port_id, strerror(rte_errno)); 341 goto error; 342 } 343 rte_wmb(); 344 dev->tx_pkt_burst = mlx5_select_tx_function(dev); 345 dev->rx_pkt_burst = mlx5_select_rx_function(dev); 346 /* Enable datapath on secondary process. */ 347 mlx5_mp_req_start_rxtx(dev); 348 if (priv->sh->intr_handle.fd >= 0) { 349 priv->sh->port[priv->dev_port - 1].ih_port_id = 350 (uint32_t)dev->data->port_id; 351 } else { 352 DRV_LOG(INFO, "port %u starts without LSC and RMV interrupts.", 353 dev->data->port_id); 354 dev->data->dev_conf.intr_conf.lsc = 0; 355 dev->data->dev_conf.intr_conf.rmv = 0; 356 } 357 if (priv->sh->intr_handle_devx.fd >= 0) 358 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = 359 (uint32_t)dev->data->port_id; 360 return 0; 361 error: 362 ret = rte_errno; /* Save rte_errno before cleanup. */ 363 /* Rollback. */ 364 dev->data->dev_started = 0; 365 mlx5_flow_stop_default(dev); 366 mlx5_traffic_disable(dev); 367 mlx5_txq_stop(dev); 368 mlx5_rxq_stop(dev); 369 mlx5_txpp_stop(dev); /* Stop last. */ 370 rte_errno = ret; /* Restore rte_errno. */ 371 return -rte_errno; 372 } 373 374 /** 375 * DPDK callback to stop the device. 376 * 377 * Simulate device stop by detaching all configured flows. 378 * 379 * @param dev 380 * Pointer to Ethernet device structure. 381 */ 382 void 383 mlx5_dev_stop(struct rte_eth_dev *dev) 384 { 385 struct mlx5_priv *priv = dev->data->dev_private; 386 387 dev->data->dev_started = 0; 388 /* Prevent crashes when queues are still in use. */ 389 dev->rx_pkt_burst = removed_rx_burst; 390 dev->tx_pkt_burst = removed_tx_burst; 391 rte_wmb(); 392 /* Disable datapath on secondary process. */ 393 mlx5_mp_req_stop_rxtx(dev); 394 usleep(1000 * priv->rxqs_n); 395 DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); 396 mlx5_flow_stop_default(dev); 397 /* Control flows for default traffic can be removed firstly. */ 398 mlx5_traffic_disable(dev); 399 /* All RX queue flags will be cleared in the flush interface. */ 400 mlx5_flow_list_flush(dev, &priv->flows, true); 401 mlx5_rx_intr_vec_disable(dev); 402 priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS; 403 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = RTE_MAX_ETHPORTS; 404 mlx5_txq_stop(dev); 405 mlx5_rxq_stop(dev); 406 mlx5_txpp_stop(dev); 407 } 408 409 /** 410 * Enable traffic flows configured by control plane 411 * 412 * @param dev 413 * Pointer to Ethernet device private data. 414 * @param dev 415 * Pointer to Ethernet device structure. 416 * 417 * @return 418 * 0 on success, a negative errno value otherwise and rte_errno is set. 419 */ 420 int 421 mlx5_traffic_enable(struct rte_eth_dev *dev) 422 { 423 struct mlx5_priv *priv = dev->data->dev_private; 424 struct rte_flow_item_eth bcast = { 425 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 426 }; 427 struct rte_flow_item_eth ipv6_multi_spec = { 428 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", 429 }; 430 struct rte_flow_item_eth ipv6_multi_mask = { 431 .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00", 432 }; 433 struct rte_flow_item_eth unicast = { 434 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 435 }; 436 struct rte_flow_item_eth unicast_mask = { 437 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 438 }; 439 const unsigned int vlan_filter_n = priv->vlan_filter_n; 440 const struct rte_ether_addr cmp = { 441 .addr_bytes = "\x00\x00\x00\x00\x00\x00", 442 }; 443 unsigned int i; 444 unsigned int j; 445 int ret; 446 447 /* 448 * Hairpin txq default flow should be created no matter if it is 449 * isolation mode. Or else all the packets to be sent will be sent 450 * out directly without the TX flow actions, e.g. encapsulation. 451 */ 452 for (i = 0; i != priv->txqs_n; ++i) { 453 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 454 if (!txq_ctrl) 455 continue; 456 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 457 ret = mlx5_ctrl_flow_source_queue(dev, i); 458 if (ret) { 459 mlx5_txq_release(dev, i); 460 goto error; 461 } 462 } 463 mlx5_txq_release(dev, i); 464 } 465 if (priv->config.dv_esw_en && !priv->config.vf) { 466 if (mlx5_flow_create_esw_table_zero_flow(dev)) 467 priv->fdb_def_rule = 1; 468 else 469 DRV_LOG(INFO, "port %u FDB default rule cannot be" 470 " configured - only Eswitch group 0 flows are" 471 " supported.", dev->data->port_id); 472 } 473 if (!priv->config.lacp_by_user && priv->pf_bond >= 0) { 474 ret = mlx5_flow_lacp_miss(dev); 475 if (ret) 476 DRV_LOG(INFO, "port %u LACP rule cannot be created - " 477 "forward LACP to kernel.", dev->data->port_id); 478 else 479 DRV_LOG(INFO, "LACP traffic will be missed in port %u." 480 , dev->data->port_id); 481 } 482 if (priv->isolated) 483 return 0; 484 if (dev->data->promiscuous) { 485 struct rte_flow_item_eth promisc = { 486 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 487 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 488 .type = 0, 489 }; 490 491 ret = mlx5_ctrl_flow(dev, &promisc, &promisc); 492 if (ret) 493 goto error; 494 } 495 if (dev->data->all_multicast) { 496 struct rte_flow_item_eth multicast = { 497 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", 498 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 499 .type = 0, 500 }; 501 502 ret = mlx5_ctrl_flow(dev, &multicast, &multicast); 503 if (ret) 504 goto error; 505 } else { 506 /* Add broadcast/multicast flows. */ 507 for (i = 0; i != vlan_filter_n; ++i) { 508 uint16_t vlan = priv->vlan_filter[i]; 509 510 struct rte_flow_item_vlan vlan_spec = { 511 .tci = rte_cpu_to_be_16(vlan), 512 }; 513 struct rte_flow_item_vlan vlan_mask = 514 rte_flow_item_vlan_mask; 515 516 ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, 517 &vlan_spec, &vlan_mask); 518 if (ret) 519 goto error; 520 ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec, 521 &ipv6_multi_mask, 522 &vlan_spec, &vlan_mask); 523 if (ret) 524 goto error; 525 } 526 if (!vlan_filter_n) { 527 ret = mlx5_ctrl_flow(dev, &bcast, &bcast); 528 if (ret) 529 goto error; 530 ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec, 531 &ipv6_multi_mask); 532 if (ret) 533 goto error; 534 } 535 } 536 /* Add MAC address flows. */ 537 for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) { 538 struct rte_ether_addr *mac = &dev->data->mac_addrs[i]; 539 540 if (!memcmp(mac, &cmp, sizeof(*mac))) 541 continue; 542 memcpy(&unicast.dst.addr_bytes, 543 mac->addr_bytes, 544 RTE_ETHER_ADDR_LEN); 545 for (j = 0; j != vlan_filter_n; ++j) { 546 uint16_t vlan = priv->vlan_filter[j]; 547 548 struct rte_flow_item_vlan vlan_spec = { 549 .tci = rte_cpu_to_be_16(vlan), 550 }; 551 struct rte_flow_item_vlan vlan_mask = 552 rte_flow_item_vlan_mask; 553 554 ret = mlx5_ctrl_flow_vlan(dev, &unicast, 555 &unicast_mask, 556 &vlan_spec, 557 &vlan_mask); 558 if (ret) 559 goto error; 560 } 561 if (!vlan_filter_n) { 562 ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask); 563 if (ret) 564 goto error; 565 } 566 } 567 return 0; 568 error: 569 ret = rte_errno; /* Save rte_errno before cleanup. */ 570 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false); 571 rte_errno = ret; /* Restore rte_errno. */ 572 return -rte_errno; 573 } 574 575 576 /** 577 * Disable traffic flows configured by control plane 578 * 579 * @param dev 580 * Pointer to Ethernet device private data. 581 */ 582 void 583 mlx5_traffic_disable(struct rte_eth_dev *dev) 584 { 585 struct mlx5_priv *priv = dev->data->dev_private; 586 587 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false); 588 } 589 590 /** 591 * Restart traffic flows configured by control plane 592 * 593 * @param dev 594 * Pointer to Ethernet device private data. 595 * 596 * @return 597 * 0 on success, a negative errno value otherwise and rte_errno is set. 598 */ 599 int 600 mlx5_traffic_restart(struct rte_eth_dev *dev) 601 { 602 if (dev->data->dev_started) { 603 mlx5_traffic_disable(dev); 604 return mlx5_traffic_enable(dev); 605 } 606 return 0; 607 } 608