1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <unistd.h> 7 8 #include <rte_ether.h> 9 #include <rte_ethdev_driver.h> 10 #include <rte_interrupts.h> 11 #include <rte_alarm.h> 12 13 #include "mlx5.h" 14 #include "mlx5_mr.h" 15 #include "mlx5_rxtx.h" 16 #include "mlx5_utils.h" 17 #include "rte_pmd_mlx5.h" 18 19 /** 20 * Stop traffic on Tx queues. 21 * 22 * @param dev 23 * Pointer to Ethernet device structure. 24 */ 25 static void 26 mlx5_txq_stop(struct rte_eth_dev *dev) 27 { 28 struct mlx5_priv *priv = dev->data->dev_private; 29 unsigned int i; 30 31 for (i = 0; i != priv->txqs_n; ++i) 32 mlx5_txq_release(dev, i); 33 } 34 35 /** 36 * Start traffic on Tx queues. 37 * 38 * @param dev 39 * Pointer to Ethernet device structure. 40 * 41 * @return 42 * 0 on success, a negative errno value otherwise and rte_errno is set. 43 */ 44 static int 45 mlx5_txq_start(struct rte_eth_dev *dev) 46 { 47 struct mlx5_priv *priv = dev->data->dev_private; 48 unsigned int i; 49 int ret; 50 51 for (i = 0; i != priv->txqs_n; ++i) { 52 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 53 54 if (!txq_ctrl) 55 continue; 56 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 57 txq_ctrl->obj = mlx5_txq_obj_new 58 (dev, i, MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN); 59 } else { 60 txq_alloc_elts(txq_ctrl); 61 txq_ctrl->obj = mlx5_txq_obj_new 62 (dev, i, MLX5_TXQ_OBJ_TYPE_IBV); 63 } 64 if (!txq_ctrl->obj) { 65 rte_errno = ENOMEM; 66 goto error; 67 } 68 } 69 return 0; 70 error: 71 ret = rte_errno; /* Save rte_errno before cleanup. */ 72 do { 73 mlx5_txq_release(dev, i); 74 } while (i-- != 0); 75 rte_errno = ret; /* Restore rte_errno. */ 76 return -rte_errno; 77 } 78 79 /** 80 * Stop traffic on Rx queues. 81 * 82 * @param dev 83 * Pointer to Ethernet device structure. 84 */ 85 static void 86 mlx5_rxq_stop(struct rte_eth_dev *dev) 87 { 88 struct mlx5_priv *priv = dev->data->dev_private; 89 unsigned int i; 90 91 for (i = 0; i != priv->rxqs_n; ++i) 92 mlx5_rxq_release(dev, i); 93 } 94 95 /** 96 * Start traffic on Rx queues. 97 * 98 * @param dev 99 * Pointer to Ethernet device structure. 100 * 101 * @return 102 * 0 on success, a negative errno value otherwise and rte_errno is set. 103 */ 104 static int 105 mlx5_rxq_start(struct rte_eth_dev *dev) 106 { 107 struct mlx5_priv *priv = dev->data->dev_private; 108 unsigned int i; 109 int ret = 0; 110 enum mlx5_rxq_obj_type obj_type = MLX5_RXQ_OBJ_TYPE_IBV; 111 struct mlx5_rxq_data *rxq = NULL; 112 113 for (i = 0; i < priv->rxqs_n; ++i) { 114 rxq = (*priv->rxqs)[i]; 115 116 if (rxq && rxq->lro) { 117 obj_type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ; 118 break; 119 } 120 } 121 /* Allocate/reuse/resize mempool for Multi-Packet RQ. */ 122 if (mlx5_mprq_alloc_mp(dev)) { 123 /* Should not release Rx queues but return immediately. */ 124 return -rte_errno; 125 } 126 for (i = 0; i != priv->rxqs_n; ++i) { 127 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i); 128 struct rte_mempool *mp; 129 130 if (!rxq_ctrl) 131 continue; 132 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) { 133 rxq_ctrl->obj = mlx5_rxq_obj_new 134 (dev, i, MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN); 135 if (!rxq_ctrl->obj) 136 goto error; 137 continue; 138 } 139 /* Pre-register Rx mempool. */ 140 mp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 141 rxq_ctrl->rxq.mprq_mp : rxq_ctrl->rxq.mp; 142 DRV_LOG(DEBUG, 143 "port %u Rx queue %u registering" 144 " mp %s having %u chunks", 145 dev->data->port_id, rxq_ctrl->rxq.idx, 146 mp->name, mp->nb_mem_chunks); 147 mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp); 148 ret = rxq_alloc_elts(rxq_ctrl); 149 if (ret) 150 goto error; 151 rxq_ctrl->obj = mlx5_rxq_obj_new(dev, i, obj_type); 152 if (!rxq_ctrl->obj) 153 goto error; 154 if (obj_type == MLX5_RXQ_OBJ_TYPE_IBV) 155 rxq_ctrl->wqn = rxq_ctrl->obj->wq->wq_num; 156 else if (obj_type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) 157 rxq_ctrl->wqn = rxq_ctrl->obj->rq->id; 158 } 159 return 0; 160 error: 161 ret = rte_errno; /* Save rte_errno before cleanup. */ 162 do { 163 mlx5_rxq_release(dev, i); 164 } while (i-- != 0); 165 rte_errno = ret; /* Restore rte_errno. */ 166 return -rte_errno; 167 } 168 169 /** 170 * Binds Tx queues to Rx queues for hairpin. 171 * 172 * Binds Tx queues to the target Rx queues. 173 * 174 * @param dev 175 * Pointer to Ethernet device structure. 176 * 177 * @return 178 * 0 on success, a negative errno value otherwise and rte_errno is set. 179 */ 180 static int 181 mlx5_hairpin_bind(struct rte_eth_dev *dev) 182 { 183 struct mlx5_priv *priv = dev->data->dev_private; 184 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 185 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 186 struct mlx5_txq_ctrl *txq_ctrl; 187 struct mlx5_rxq_ctrl *rxq_ctrl; 188 struct mlx5_devx_obj *sq; 189 struct mlx5_devx_obj *rq; 190 unsigned int i; 191 int ret = 0; 192 193 for (i = 0; i != priv->txqs_n; ++i) { 194 txq_ctrl = mlx5_txq_get(dev, i); 195 if (!txq_ctrl) 196 continue; 197 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 198 mlx5_txq_release(dev, i); 199 continue; 200 } 201 if (!txq_ctrl->obj) { 202 rte_errno = ENOMEM; 203 DRV_LOG(ERR, "port %u no txq object found: %d", 204 dev->data->port_id, i); 205 mlx5_txq_release(dev, i); 206 return -rte_errno; 207 } 208 sq = txq_ctrl->obj->sq; 209 rxq_ctrl = mlx5_rxq_get(dev, 210 txq_ctrl->hairpin_conf.peers[0].queue); 211 if (!rxq_ctrl) { 212 mlx5_txq_release(dev, i); 213 rte_errno = EINVAL; 214 DRV_LOG(ERR, "port %u no rxq object found: %d", 215 dev->data->port_id, 216 txq_ctrl->hairpin_conf.peers[0].queue); 217 return -rte_errno; 218 } 219 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN || 220 rxq_ctrl->hairpin_conf.peers[0].queue != i) { 221 rte_errno = ENOMEM; 222 DRV_LOG(ERR, "port %u Tx queue %d can't be binded to " 223 "Rx queue %d", dev->data->port_id, 224 i, txq_ctrl->hairpin_conf.peers[0].queue); 225 goto error; 226 } 227 rq = rxq_ctrl->obj->rq; 228 if (!rq) { 229 rte_errno = ENOMEM; 230 DRV_LOG(ERR, "port %u hairpin no matching rxq: %d", 231 dev->data->port_id, 232 txq_ctrl->hairpin_conf.peers[0].queue); 233 goto error; 234 } 235 sq_attr.state = MLX5_SQC_STATE_RDY; 236 sq_attr.sq_state = MLX5_SQC_STATE_RST; 237 sq_attr.hairpin_peer_rq = rq->id; 238 sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 239 ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr); 240 if (ret) 241 goto error; 242 rq_attr.state = MLX5_SQC_STATE_RDY; 243 rq_attr.rq_state = MLX5_SQC_STATE_RST; 244 rq_attr.hairpin_peer_sq = sq->id; 245 rq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 246 ret = mlx5_devx_cmd_modify_rq(rq, &rq_attr); 247 if (ret) 248 goto error; 249 mlx5_txq_release(dev, i); 250 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 251 } 252 return 0; 253 error: 254 mlx5_txq_release(dev, i); 255 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 256 return -rte_errno; 257 } 258 259 /** 260 * DPDK callback to start the device. 261 * 262 * Simulate device start by attaching all configured flows. 263 * 264 * @param dev 265 * Pointer to Ethernet device structure. 266 * 267 * @return 268 * 0 on success, a negative errno value otherwise and rte_errno is set. 269 */ 270 int 271 mlx5_dev_start(struct rte_eth_dev *dev) 272 { 273 int ret; 274 int fine_inline; 275 276 DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); 277 fine_inline = rte_mbuf_dynflag_lookup 278 (RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, NULL); 279 if (fine_inline > 0) 280 rte_net_mlx5_dynf_inline_mask = 1UL << fine_inline; 281 else 282 rte_net_mlx5_dynf_inline_mask = 0; 283 if (dev->data->nb_rx_queues > 0) { 284 ret = mlx5_dev_configure_rss_reta(dev); 285 if (ret) { 286 DRV_LOG(ERR, "port %u reta config failed: %s", 287 dev->data->port_id, strerror(rte_errno)); 288 return -rte_errno; 289 } 290 } 291 ret = mlx5_txq_start(dev); 292 if (ret) { 293 DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", 294 dev->data->port_id, strerror(rte_errno)); 295 return -rte_errno; 296 } 297 ret = mlx5_rxq_start(dev); 298 if (ret) { 299 DRV_LOG(ERR, "port %u Rx queue allocation failed: %s", 300 dev->data->port_id, strerror(rte_errno)); 301 mlx5_txq_stop(dev); 302 return -rte_errno; 303 } 304 ret = mlx5_hairpin_bind(dev); 305 if (ret) { 306 DRV_LOG(ERR, "port %u hairpin binding failed: %s", 307 dev->data->port_id, strerror(rte_errno)); 308 mlx5_txq_stop(dev); 309 return -rte_errno; 310 } 311 /* Set started flag here for the following steps like control flow. */ 312 dev->data->dev_started = 1; 313 ret = mlx5_rx_intr_vec_enable(dev); 314 if (ret) { 315 DRV_LOG(ERR, "port %u Rx interrupt vector creation failed", 316 dev->data->port_id); 317 goto error; 318 } 319 mlx5_stats_init(dev); 320 ret = mlx5_traffic_enable(dev); 321 if (ret) { 322 DRV_LOG(ERR, "port %u failed to set defaults flows", 323 dev->data->port_id); 324 goto error; 325 } 326 /* 327 * In non-cached mode, it only needs to start the default mreg copy 328 * action and no flow created by application exists anymore. 329 * But it is worth wrapping the interface for further usage. 330 */ 331 ret = mlx5_flow_start_default(dev); 332 if (ret) { 333 DRV_LOG(DEBUG, "port %u failed to start default actions: %s", 334 dev->data->port_id, strerror(rte_errno)); 335 goto error; 336 } 337 rte_wmb(); 338 dev->tx_pkt_burst = mlx5_select_tx_function(dev); 339 dev->rx_pkt_burst = mlx5_select_rx_function(dev); 340 /* Enable datapath on secondary process. */ 341 mlx5_mp_req_start_rxtx(dev); 342 mlx5_dev_interrupt_handler_install(dev); 343 return 0; 344 error: 345 ret = rte_errno; /* Save rte_errno before cleanup. */ 346 /* Rollback. */ 347 dev->data->dev_started = 0; 348 mlx5_flow_stop_default(dev); 349 mlx5_traffic_disable(dev); 350 mlx5_txq_stop(dev); 351 mlx5_rxq_stop(dev); 352 rte_errno = ret; /* Restore rte_errno. */ 353 return -rte_errno; 354 } 355 356 /** 357 * DPDK callback to stop the device. 358 * 359 * Simulate device stop by detaching all configured flows. 360 * 361 * @param dev 362 * Pointer to Ethernet device structure. 363 */ 364 void 365 mlx5_dev_stop(struct rte_eth_dev *dev) 366 { 367 struct mlx5_priv *priv = dev->data->dev_private; 368 369 dev->data->dev_started = 0; 370 /* Prevent crashes when queues are still in use. */ 371 dev->rx_pkt_burst = removed_rx_burst; 372 dev->tx_pkt_burst = removed_tx_burst; 373 rte_wmb(); 374 /* Disable datapath on secondary process. */ 375 mlx5_mp_req_stop_rxtx(dev); 376 usleep(1000 * priv->rxqs_n); 377 DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); 378 mlx5_flow_stop_default(dev); 379 /* Control flows for default traffic can be removed firstly. */ 380 mlx5_traffic_disable(dev); 381 /* All RX queue flags will be cleared in the flush interface. */ 382 mlx5_flow_list_flush(dev, &priv->flows, true); 383 mlx5_rx_intr_vec_disable(dev); 384 mlx5_dev_interrupt_handler_uninstall(dev); 385 mlx5_txq_stop(dev); 386 mlx5_rxq_stop(dev); 387 } 388 389 /** 390 * Enable traffic flows configured by control plane 391 * 392 * @param dev 393 * Pointer to Ethernet device private data. 394 * @param dev 395 * Pointer to Ethernet device structure. 396 * 397 * @return 398 * 0 on success, a negative errno value otherwise and rte_errno is set. 399 */ 400 int 401 mlx5_traffic_enable(struct rte_eth_dev *dev) 402 { 403 struct mlx5_priv *priv = dev->data->dev_private; 404 struct rte_flow_item_eth bcast = { 405 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 406 }; 407 struct rte_flow_item_eth ipv6_multi_spec = { 408 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", 409 }; 410 struct rte_flow_item_eth ipv6_multi_mask = { 411 .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00", 412 }; 413 struct rte_flow_item_eth unicast = { 414 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 415 }; 416 struct rte_flow_item_eth unicast_mask = { 417 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 418 }; 419 const unsigned int vlan_filter_n = priv->vlan_filter_n; 420 const struct rte_ether_addr cmp = { 421 .addr_bytes = "\x00\x00\x00\x00\x00\x00", 422 }; 423 unsigned int i; 424 unsigned int j; 425 int ret; 426 427 /* 428 * Hairpin txq default flow should be created no matter if it is 429 * isolation mode. Or else all the packets to be sent will be sent 430 * out directly without the TX flow actions, e.g. encapsulation. 431 */ 432 for (i = 0; i != priv->txqs_n; ++i) { 433 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 434 if (!txq_ctrl) 435 continue; 436 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 437 ret = mlx5_ctrl_flow_source_queue(dev, i); 438 if (ret) { 439 mlx5_txq_release(dev, i); 440 goto error; 441 } 442 } 443 mlx5_txq_release(dev, i); 444 } 445 if (priv->config.dv_esw_en && !priv->config.vf) { 446 if (mlx5_flow_create_esw_table_zero_flow(dev)) 447 priv->fdb_def_rule = 1; 448 else 449 DRV_LOG(INFO, "port %u FDB default rule cannot be" 450 " configured - only Eswitch group 0 flows are" 451 " supported.", dev->data->port_id); 452 } 453 if (priv->isolated) 454 return 0; 455 if (dev->data->promiscuous) { 456 struct rte_flow_item_eth promisc = { 457 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 458 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 459 .type = 0, 460 }; 461 462 ret = mlx5_ctrl_flow(dev, &promisc, &promisc); 463 if (ret) 464 goto error; 465 } 466 if (dev->data->all_multicast) { 467 struct rte_flow_item_eth multicast = { 468 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", 469 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 470 .type = 0, 471 }; 472 473 ret = mlx5_ctrl_flow(dev, &multicast, &multicast); 474 if (ret) 475 goto error; 476 } else { 477 /* Add broadcast/multicast flows. */ 478 for (i = 0; i != vlan_filter_n; ++i) { 479 uint16_t vlan = priv->vlan_filter[i]; 480 481 struct rte_flow_item_vlan vlan_spec = { 482 .tci = rte_cpu_to_be_16(vlan), 483 }; 484 struct rte_flow_item_vlan vlan_mask = 485 rte_flow_item_vlan_mask; 486 487 ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, 488 &vlan_spec, &vlan_mask); 489 if (ret) 490 goto error; 491 ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec, 492 &ipv6_multi_mask, 493 &vlan_spec, &vlan_mask); 494 if (ret) 495 goto error; 496 } 497 if (!vlan_filter_n) { 498 ret = mlx5_ctrl_flow(dev, &bcast, &bcast); 499 if (ret) 500 goto error; 501 ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec, 502 &ipv6_multi_mask); 503 if (ret) 504 goto error; 505 } 506 } 507 /* Add MAC address flows. */ 508 for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) { 509 struct rte_ether_addr *mac = &dev->data->mac_addrs[i]; 510 511 if (!memcmp(mac, &cmp, sizeof(*mac))) 512 continue; 513 memcpy(&unicast.dst.addr_bytes, 514 mac->addr_bytes, 515 RTE_ETHER_ADDR_LEN); 516 for (j = 0; j != vlan_filter_n; ++j) { 517 uint16_t vlan = priv->vlan_filter[j]; 518 519 struct rte_flow_item_vlan vlan_spec = { 520 .tci = rte_cpu_to_be_16(vlan), 521 }; 522 struct rte_flow_item_vlan vlan_mask = 523 rte_flow_item_vlan_mask; 524 525 ret = mlx5_ctrl_flow_vlan(dev, &unicast, 526 &unicast_mask, 527 &vlan_spec, 528 &vlan_mask); 529 if (ret) 530 goto error; 531 } 532 if (!vlan_filter_n) { 533 ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask); 534 if (ret) 535 goto error; 536 } 537 } 538 return 0; 539 error: 540 ret = rte_errno; /* Save rte_errno before cleanup. */ 541 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false); 542 rte_errno = ret; /* Restore rte_errno. */ 543 return -rte_errno; 544 } 545 546 547 /** 548 * Disable traffic flows configured by control plane 549 * 550 * @param dev 551 * Pointer to Ethernet device private data. 552 */ 553 void 554 mlx5_traffic_disable(struct rte_eth_dev *dev) 555 { 556 struct mlx5_priv *priv = dev->data->dev_private; 557 558 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false); 559 } 560 561 /** 562 * Restart traffic flows configured by control plane 563 * 564 * @param dev 565 * Pointer to Ethernet device private data. 566 * 567 * @return 568 * 0 on success, a negative errno value otherwise and rte_errno is set. 569 */ 570 int 571 mlx5_traffic_restart(struct rte_eth_dev *dev) 572 { 573 if (dev->data->dev_started) { 574 mlx5_traffic_disable(dev); 575 return mlx5_traffic_enable(dev); 576 } 577 return 0; 578 } 579