1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2015 6WIND S.A. 3 * Copyright 2015 Mellanox Technologies, Ltd 4 */ 5 6 #include <unistd.h> 7 8 #include <rte_ether.h> 9 #include <rte_ethdev_driver.h> 10 #include <rte_interrupts.h> 11 #include <rte_alarm.h> 12 13 #include "mlx5.h" 14 #include "mlx5_rxtx.h" 15 #include "mlx5_utils.h" 16 #include "rte_pmd_mlx5.h" 17 18 /** 19 * Stop traffic on Tx queues. 20 * 21 * @param dev 22 * Pointer to Ethernet device structure. 23 */ 24 static void 25 mlx5_txq_stop(struct rte_eth_dev *dev) 26 { 27 struct mlx5_priv *priv = dev->data->dev_private; 28 unsigned int i; 29 30 for (i = 0; i != priv->txqs_n; ++i) 31 mlx5_txq_release(dev, i); 32 } 33 34 /** 35 * Start traffic on Tx queues. 36 * 37 * @param dev 38 * Pointer to Ethernet device structure. 39 * 40 * @return 41 * 0 on success, a negative errno value otherwise and rte_errno is set. 42 */ 43 static int 44 mlx5_txq_start(struct rte_eth_dev *dev) 45 { 46 struct mlx5_priv *priv = dev->data->dev_private; 47 unsigned int i; 48 int ret; 49 50 for (i = 0; i != priv->txqs_n; ++i) { 51 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 52 53 if (!txq_ctrl) 54 continue; 55 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 56 txq_ctrl->obj = mlx5_txq_obj_new 57 (dev, i, MLX5_TXQ_OBJ_TYPE_DEVX_HAIRPIN); 58 } else { 59 txq_alloc_elts(txq_ctrl); 60 txq_ctrl->obj = mlx5_txq_obj_new 61 (dev, i, MLX5_TXQ_OBJ_TYPE_IBV); 62 } 63 if (!txq_ctrl->obj) { 64 rte_errno = ENOMEM; 65 goto error; 66 } 67 } 68 return 0; 69 error: 70 ret = rte_errno; /* Save rte_errno before cleanup. */ 71 do { 72 mlx5_txq_release(dev, i); 73 } while (i-- != 0); 74 rte_errno = ret; /* Restore rte_errno. */ 75 return -rte_errno; 76 } 77 78 /** 79 * Stop traffic on Rx queues. 80 * 81 * @param dev 82 * Pointer to Ethernet device structure. 83 */ 84 static void 85 mlx5_rxq_stop(struct rte_eth_dev *dev) 86 { 87 struct mlx5_priv *priv = dev->data->dev_private; 88 unsigned int i; 89 90 for (i = 0; i != priv->rxqs_n; ++i) 91 mlx5_rxq_release(dev, i); 92 } 93 94 /** 95 * Start traffic on Rx queues. 96 * 97 * @param dev 98 * Pointer to Ethernet device structure. 99 * 100 * @return 101 * 0 on success, a negative errno value otherwise and rte_errno is set. 102 */ 103 static int 104 mlx5_rxq_start(struct rte_eth_dev *dev) 105 { 106 struct mlx5_priv *priv = dev->data->dev_private; 107 unsigned int i; 108 int ret = 0; 109 enum mlx5_rxq_obj_type obj_type = MLX5_RXQ_OBJ_TYPE_IBV; 110 struct mlx5_rxq_data *rxq = NULL; 111 112 for (i = 0; i < priv->rxqs_n; ++i) { 113 rxq = (*priv->rxqs)[i]; 114 115 if (rxq && rxq->lro) { 116 obj_type = MLX5_RXQ_OBJ_TYPE_DEVX_RQ; 117 break; 118 } 119 } 120 /* Allocate/reuse/resize mempool for Multi-Packet RQ. */ 121 if (mlx5_mprq_alloc_mp(dev)) { 122 /* Should not release Rx queues but return immediately. */ 123 return -rte_errno; 124 } 125 for (i = 0; i != priv->rxqs_n; ++i) { 126 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i); 127 struct rte_mempool *mp; 128 129 if (!rxq_ctrl) 130 continue; 131 if (rxq_ctrl->type == MLX5_RXQ_TYPE_HAIRPIN) { 132 rxq_ctrl->obj = mlx5_rxq_obj_new 133 (dev, i, MLX5_RXQ_OBJ_TYPE_DEVX_HAIRPIN); 134 if (!rxq_ctrl->obj) 135 goto error; 136 continue; 137 } 138 /* Pre-register Rx mempool. */ 139 mp = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? 140 rxq_ctrl->rxq.mprq_mp : rxq_ctrl->rxq.mp; 141 DRV_LOG(DEBUG, 142 "port %u Rx queue %u registering" 143 " mp %s having %u chunks", 144 dev->data->port_id, rxq_ctrl->rxq.idx, 145 mp->name, mp->nb_mem_chunks); 146 mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp); 147 ret = rxq_alloc_elts(rxq_ctrl); 148 if (ret) 149 goto error; 150 rxq_ctrl->obj = mlx5_rxq_obj_new(dev, i, obj_type); 151 if (!rxq_ctrl->obj) 152 goto error; 153 if (obj_type == MLX5_RXQ_OBJ_TYPE_IBV) 154 rxq_ctrl->wqn = rxq_ctrl->obj->wq->wq_num; 155 else if (obj_type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) 156 rxq_ctrl->wqn = rxq_ctrl->obj->rq->id; 157 } 158 return 0; 159 error: 160 ret = rte_errno; /* Save rte_errno before cleanup. */ 161 do { 162 mlx5_rxq_release(dev, i); 163 } while (i-- != 0); 164 rte_errno = ret; /* Restore rte_errno. */ 165 return -rte_errno; 166 } 167 168 /** 169 * Binds Tx queues to Rx queues for hairpin. 170 * 171 * Binds Tx queues to the target Rx queues. 172 * 173 * @param dev 174 * Pointer to Ethernet device structure. 175 * 176 * @return 177 * 0 on success, a negative errno value otherwise and rte_errno is set. 178 */ 179 static int 180 mlx5_hairpin_bind(struct rte_eth_dev *dev) 181 { 182 struct mlx5_priv *priv = dev->data->dev_private; 183 struct mlx5_devx_modify_sq_attr sq_attr = { 0 }; 184 struct mlx5_devx_modify_rq_attr rq_attr = { 0 }; 185 struct mlx5_txq_ctrl *txq_ctrl; 186 struct mlx5_rxq_ctrl *rxq_ctrl; 187 struct mlx5_devx_obj *sq; 188 struct mlx5_devx_obj *rq; 189 unsigned int i; 190 int ret = 0; 191 192 for (i = 0; i != priv->txqs_n; ++i) { 193 txq_ctrl = mlx5_txq_get(dev, i); 194 if (!txq_ctrl) 195 continue; 196 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) { 197 mlx5_txq_release(dev, i); 198 continue; 199 } 200 if (!txq_ctrl->obj) { 201 rte_errno = ENOMEM; 202 DRV_LOG(ERR, "port %u no txq object found: %d", 203 dev->data->port_id, i); 204 mlx5_txq_release(dev, i); 205 return -rte_errno; 206 } 207 sq = txq_ctrl->obj->sq; 208 rxq_ctrl = mlx5_rxq_get(dev, 209 txq_ctrl->hairpin_conf.peers[0].queue); 210 if (!rxq_ctrl) { 211 mlx5_txq_release(dev, i); 212 rte_errno = EINVAL; 213 DRV_LOG(ERR, "port %u no rxq object found: %d", 214 dev->data->port_id, 215 txq_ctrl->hairpin_conf.peers[0].queue); 216 return -rte_errno; 217 } 218 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN || 219 rxq_ctrl->hairpin_conf.peers[0].queue != i) { 220 rte_errno = ENOMEM; 221 DRV_LOG(ERR, "port %u Tx queue %d can't be binded to " 222 "Rx queue %d", dev->data->port_id, 223 i, txq_ctrl->hairpin_conf.peers[0].queue); 224 goto error; 225 } 226 rq = rxq_ctrl->obj->rq; 227 if (!rq) { 228 rte_errno = ENOMEM; 229 DRV_LOG(ERR, "port %u hairpin no matching rxq: %d", 230 dev->data->port_id, 231 txq_ctrl->hairpin_conf.peers[0].queue); 232 goto error; 233 } 234 sq_attr.state = MLX5_SQC_STATE_RDY; 235 sq_attr.sq_state = MLX5_SQC_STATE_RST; 236 sq_attr.hairpin_peer_rq = rq->id; 237 sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 238 ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr); 239 if (ret) 240 goto error; 241 rq_attr.state = MLX5_SQC_STATE_RDY; 242 rq_attr.rq_state = MLX5_SQC_STATE_RST; 243 rq_attr.hairpin_peer_sq = sq->id; 244 rq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id; 245 ret = mlx5_devx_cmd_modify_rq(rq, &rq_attr); 246 if (ret) 247 goto error; 248 mlx5_txq_release(dev, i); 249 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 250 } 251 return 0; 252 error: 253 mlx5_txq_release(dev, i); 254 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue); 255 return -rte_errno; 256 } 257 258 /** 259 * DPDK callback to start the device. 260 * 261 * Simulate device start by attaching all configured flows. 262 * 263 * @param dev 264 * Pointer to Ethernet device structure. 265 * 266 * @return 267 * 0 on success, a negative errno value otherwise and rte_errno is set. 268 */ 269 int 270 mlx5_dev_start(struct rte_eth_dev *dev) 271 { 272 struct mlx5_priv *priv = dev->data->dev_private; 273 int ret; 274 int fine_inline; 275 276 DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); 277 fine_inline = rte_mbuf_dynflag_lookup 278 (RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, NULL); 279 if (fine_inline > 0) 280 rte_net_mlx5_dynf_inline_mask = 1UL << fine_inline; 281 else 282 rte_net_mlx5_dynf_inline_mask = 0; 283 ret = mlx5_dev_configure_rss_reta(dev); 284 if (ret) { 285 DRV_LOG(ERR, "port %u reta config failed: %s", 286 dev->data->port_id, strerror(rte_errno)); 287 return -rte_errno; 288 } 289 ret = mlx5_txq_start(dev); 290 if (ret) { 291 DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", 292 dev->data->port_id, strerror(rte_errno)); 293 return -rte_errno; 294 } 295 ret = mlx5_rxq_start(dev); 296 if (ret) { 297 DRV_LOG(ERR, "port %u Rx queue allocation failed: %s", 298 dev->data->port_id, strerror(rte_errno)); 299 mlx5_txq_stop(dev); 300 return -rte_errno; 301 } 302 ret = mlx5_hairpin_bind(dev); 303 if (ret) { 304 DRV_LOG(ERR, "port %u hairpin binding failed: %s", 305 dev->data->port_id, strerror(rte_errno)); 306 mlx5_txq_stop(dev); 307 return -rte_errno; 308 } 309 dev->data->dev_started = 1; 310 ret = mlx5_rx_intr_vec_enable(dev); 311 if (ret) { 312 DRV_LOG(ERR, "port %u Rx interrupt vector creation failed", 313 dev->data->port_id); 314 goto error; 315 } 316 mlx5_stats_init(dev); 317 ret = mlx5_traffic_enable(dev); 318 if (ret) { 319 DRV_LOG(DEBUG, "port %u failed to set defaults flows", 320 dev->data->port_id); 321 goto error; 322 } 323 ret = mlx5_flow_start(dev, &priv->flows); 324 if (ret) { 325 DRV_LOG(DEBUG, "port %u failed to set flows", 326 dev->data->port_id); 327 goto error; 328 } 329 rte_wmb(); 330 dev->tx_pkt_burst = mlx5_select_tx_function(dev); 331 dev->rx_pkt_burst = mlx5_select_rx_function(dev); 332 /* Enable datapath on secondary process. */ 333 mlx5_mp_req_start_rxtx(dev); 334 mlx5_dev_interrupt_handler_install(dev); 335 return 0; 336 error: 337 ret = rte_errno; /* Save rte_errno before cleanup. */ 338 /* Rollback. */ 339 dev->data->dev_started = 0; 340 mlx5_flow_stop(dev, &priv->flows); 341 mlx5_traffic_disable(dev); 342 mlx5_txq_stop(dev); 343 mlx5_rxq_stop(dev); 344 rte_errno = ret; /* Restore rte_errno. */ 345 return -rte_errno; 346 } 347 348 /** 349 * DPDK callback to stop the device. 350 * 351 * Simulate device stop by detaching all configured flows. 352 * 353 * @param dev 354 * Pointer to Ethernet device structure. 355 */ 356 void 357 mlx5_dev_stop(struct rte_eth_dev *dev) 358 { 359 struct mlx5_priv *priv = dev->data->dev_private; 360 361 dev->data->dev_started = 0; 362 /* Prevent crashes when queues are still in use. */ 363 dev->rx_pkt_burst = removed_rx_burst; 364 dev->tx_pkt_burst = removed_tx_burst; 365 rte_wmb(); 366 /* Disable datapath on secondary process. */ 367 mlx5_mp_req_stop_rxtx(dev); 368 usleep(1000 * priv->rxqs_n); 369 DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); 370 mlx5_flow_stop(dev, &priv->flows); 371 mlx5_traffic_disable(dev); 372 mlx5_rx_intr_vec_disable(dev); 373 mlx5_dev_interrupt_handler_uninstall(dev); 374 mlx5_txq_stop(dev); 375 mlx5_rxq_stop(dev); 376 } 377 378 /** 379 * Enable traffic flows configured by control plane 380 * 381 * @param dev 382 * Pointer to Ethernet device private data. 383 * @param dev 384 * Pointer to Ethernet device structure. 385 * 386 * @return 387 * 0 on success, a negative errno value otherwise and rte_errno is set. 388 */ 389 int 390 mlx5_traffic_enable(struct rte_eth_dev *dev) 391 { 392 struct mlx5_priv *priv = dev->data->dev_private; 393 struct rte_flow_item_eth bcast = { 394 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 395 }; 396 struct rte_flow_item_eth ipv6_multi_spec = { 397 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00", 398 }; 399 struct rte_flow_item_eth ipv6_multi_mask = { 400 .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00", 401 }; 402 struct rte_flow_item_eth unicast = { 403 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 404 }; 405 struct rte_flow_item_eth unicast_mask = { 406 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", 407 }; 408 const unsigned int vlan_filter_n = priv->vlan_filter_n; 409 const struct rte_ether_addr cmp = { 410 .addr_bytes = "\x00\x00\x00\x00\x00\x00", 411 }; 412 unsigned int i; 413 unsigned int j; 414 int ret; 415 416 /* 417 * Hairpin txq default flow should be created no matter if it is 418 * isolation mode. Or else all the packets to be sent will be sent 419 * out directly without the TX flow actions, e.g. encapsulation. 420 */ 421 for (i = 0; i != priv->txqs_n; ++i) { 422 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); 423 if (!txq_ctrl) 424 continue; 425 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN) { 426 ret = mlx5_ctrl_flow_source_queue(dev, i); 427 if (ret) { 428 mlx5_txq_release(dev, i); 429 goto error; 430 } 431 } 432 mlx5_txq_release(dev, i); 433 } 434 if (priv->config.dv_esw_en && !priv->config.vf) { 435 if (mlx5_flow_create_esw_table_zero_flow(dev)) 436 priv->fdb_def_rule = 1; 437 else 438 DRV_LOG(INFO, "port %u FDB default rule cannot be" 439 " configured - only Eswitch group 0 flows are" 440 " supported.", dev->data->port_id); 441 } 442 if (priv->isolated) 443 return 0; 444 if (dev->data->promiscuous) { 445 struct rte_flow_item_eth promisc = { 446 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00", 447 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 448 .type = 0, 449 }; 450 451 ret = mlx5_ctrl_flow(dev, &promisc, &promisc); 452 if (ret) 453 goto error; 454 } 455 if (dev->data->all_multicast) { 456 struct rte_flow_item_eth multicast = { 457 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00", 458 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00", 459 .type = 0, 460 }; 461 462 ret = mlx5_ctrl_flow(dev, &multicast, &multicast); 463 if (ret) 464 goto error; 465 } else { 466 /* Add broadcast/multicast flows. */ 467 for (i = 0; i != vlan_filter_n; ++i) { 468 uint16_t vlan = priv->vlan_filter[i]; 469 470 struct rte_flow_item_vlan vlan_spec = { 471 .tci = rte_cpu_to_be_16(vlan), 472 }; 473 struct rte_flow_item_vlan vlan_mask = 474 rte_flow_item_vlan_mask; 475 476 ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, 477 &vlan_spec, &vlan_mask); 478 if (ret) 479 goto error; 480 ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec, 481 &ipv6_multi_mask, 482 &vlan_spec, &vlan_mask); 483 if (ret) 484 goto error; 485 } 486 if (!vlan_filter_n) { 487 ret = mlx5_ctrl_flow(dev, &bcast, &bcast); 488 if (ret) 489 goto error; 490 ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec, 491 &ipv6_multi_mask); 492 if (ret) 493 goto error; 494 } 495 } 496 /* Add MAC address flows. */ 497 for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) { 498 struct rte_ether_addr *mac = &dev->data->mac_addrs[i]; 499 500 if (!memcmp(mac, &cmp, sizeof(*mac))) 501 continue; 502 memcpy(&unicast.dst.addr_bytes, 503 mac->addr_bytes, 504 RTE_ETHER_ADDR_LEN); 505 for (j = 0; j != vlan_filter_n; ++j) { 506 uint16_t vlan = priv->vlan_filter[j]; 507 508 struct rte_flow_item_vlan vlan_spec = { 509 .tci = rte_cpu_to_be_16(vlan), 510 }; 511 struct rte_flow_item_vlan vlan_mask = 512 rte_flow_item_vlan_mask; 513 514 ret = mlx5_ctrl_flow_vlan(dev, &unicast, 515 &unicast_mask, 516 &vlan_spec, 517 &vlan_mask); 518 if (ret) 519 goto error; 520 } 521 if (!vlan_filter_n) { 522 ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask); 523 if (ret) 524 goto error; 525 } 526 } 527 return 0; 528 error: 529 ret = rte_errno; /* Save rte_errno before cleanup. */ 530 mlx5_flow_list_flush(dev, &priv->ctrl_flows); 531 rte_errno = ret; /* Restore rte_errno. */ 532 return -rte_errno; 533 } 534 535 536 /** 537 * Disable traffic flows configured by control plane 538 * 539 * @param dev 540 * Pointer to Ethernet device private data. 541 */ 542 void 543 mlx5_traffic_disable(struct rte_eth_dev *dev) 544 { 545 struct mlx5_priv *priv = dev->data->dev_private; 546 547 mlx5_flow_list_flush(dev, &priv->ctrl_flows); 548 } 549 550 /** 551 * Restart traffic flows configured by control plane 552 * 553 * @param dev 554 * Pointer to Ethernet device private data. 555 * 556 * @return 557 * 0 on success, a negative errno value otherwise and rte_errno is set. 558 */ 559 int 560 mlx5_traffic_restart(struct rte_eth_dev *dev) 561 { 562 if (dev->data->dev_started) { 563 mlx5_traffic_disable(dev); 564 return mlx5_traffic_enable(dev); 565 } 566 return 0; 567 } 568