1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdbool.h> 7 #include <stdint.h> 8 #include <unistd.h> 9 10 #include <rte_debug.h> 11 #include <rte_atomic.h> 12 #include <rte_ethdev_driver.h> 13 #include <rte_malloc.h> 14 #include <rte_flow.h> 15 #include <rte_cycles.h> 16 #include <rte_ethdev.h> 17 18 #include "failsafe_private.h" 19 20 static struct rte_eth_dev_info default_infos = { 21 /* Max possible number of elements */ 22 .max_rx_pktlen = UINT32_MAX, 23 .max_rx_queues = RTE_MAX_QUEUES_PER_PORT, 24 .max_tx_queues = RTE_MAX_QUEUES_PER_PORT, 25 .max_mac_addrs = FAILSAFE_MAX_ETHADDR, 26 .max_hash_mac_addrs = UINT32_MAX, 27 .max_vfs = UINT16_MAX, 28 .max_vmdq_pools = UINT16_MAX, 29 .rx_desc_lim = { 30 .nb_max = UINT16_MAX, 31 .nb_min = 0, 32 .nb_align = 1, 33 .nb_seg_max = UINT16_MAX, 34 .nb_mtu_seg_max = UINT16_MAX, 35 }, 36 .tx_desc_lim = { 37 .nb_max = UINT16_MAX, 38 .nb_min = 0, 39 .nb_align = 1, 40 .nb_seg_max = UINT16_MAX, 41 .nb_mtu_seg_max = UINT16_MAX, 42 }, 43 /* 44 * Set of capabilities that can be verified upon 45 * configuring a sub-device. 46 */ 47 .rx_offload_capa = 48 DEV_RX_OFFLOAD_VLAN_STRIP | 49 DEV_RX_OFFLOAD_IPV4_CKSUM | 50 DEV_RX_OFFLOAD_UDP_CKSUM | 51 DEV_RX_OFFLOAD_TCP_CKSUM | 52 DEV_RX_OFFLOAD_TCP_LRO | 53 DEV_RX_OFFLOAD_QINQ_STRIP | 54 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 55 DEV_RX_OFFLOAD_MACSEC_STRIP | 56 DEV_RX_OFFLOAD_HEADER_SPLIT | 57 DEV_RX_OFFLOAD_VLAN_FILTER | 58 DEV_RX_OFFLOAD_VLAN_EXTEND | 59 DEV_RX_OFFLOAD_JUMBO_FRAME | 60 DEV_RX_OFFLOAD_SCATTER | 61 DEV_RX_OFFLOAD_TIMESTAMP | 62 DEV_RX_OFFLOAD_SECURITY, 63 .rx_queue_offload_capa = 64 DEV_RX_OFFLOAD_VLAN_STRIP | 65 DEV_RX_OFFLOAD_IPV4_CKSUM | 66 DEV_RX_OFFLOAD_UDP_CKSUM | 67 DEV_RX_OFFLOAD_TCP_CKSUM | 68 DEV_RX_OFFLOAD_TCP_LRO | 69 DEV_RX_OFFLOAD_QINQ_STRIP | 70 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 71 DEV_RX_OFFLOAD_MACSEC_STRIP | 72 DEV_RX_OFFLOAD_HEADER_SPLIT | 73 DEV_RX_OFFLOAD_VLAN_FILTER | 74 DEV_RX_OFFLOAD_VLAN_EXTEND | 75 DEV_RX_OFFLOAD_JUMBO_FRAME | 76 DEV_RX_OFFLOAD_SCATTER | 77 DEV_RX_OFFLOAD_TIMESTAMP | 78 DEV_RX_OFFLOAD_SECURITY, 79 .tx_offload_capa = 80 DEV_TX_OFFLOAD_MULTI_SEGS | 81 DEV_TX_OFFLOAD_IPV4_CKSUM | 82 DEV_TX_OFFLOAD_UDP_CKSUM | 83 DEV_TX_OFFLOAD_TCP_CKSUM | 84 DEV_TX_OFFLOAD_TCP_TSO, 85 .flow_type_rss_offloads = 86 ETH_RSS_IP | 87 ETH_RSS_UDP | 88 ETH_RSS_TCP, 89 }; 90 91 static int 92 fs_dev_configure(struct rte_eth_dev *dev) 93 { 94 struct sub_device *sdev; 95 uint8_t i; 96 int ret; 97 98 fs_lock(dev, 0); 99 FOREACH_SUBDEV(sdev, i, dev) { 100 int rmv_interrupt = 0; 101 int lsc_interrupt = 0; 102 int lsc_enabled; 103 104 if (sdev->state != DEV_PROBED && 105 !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE)) 106 continue; 107 108 rmv_interrupt = ETH(sdev)->data->dev_flags & 109 RTE_ETH_DEV_INTR_RMV; 110 if (rmv_interrupt) { 111 DEBUG("Enabling RMV interrupts for sub_device %d", i); 112 dev->data->dev_conf.intr_conf.rmv = 1; 113 } else { 114 DEBUG("sub_device %d does not support RMV event", i); 115 } 116 lsc_enabled = dev->data->dev_conf.intr_conf.lsc; 117 lsc_interrupt = lsc_enabled && 118 (ETH(sdev)->data->dev_flags & 119 RTE_ETH_DEV_INTR_LSC); 120 if (lsc_interrupt) { 121 DEBUG("Enabling LSC interrupts for sub_device %d", i); 122 dev->data->dev_conf.intr_conf.lsc = 1; 123 } else if (lsc_enabled && !lsc_interrupt) { 124 DEBUG("Disabling LSC interrupts for sub_device %d", i); 125 dev->data->dev_conf.intr_conf.lsc = 0; 126 } 127 DEBUG("Configuring sub-device %d", i); 128 ret = rte_eth_dev_configure(PORT_ID(sdev), 129 dev->data->nb_rx_queues, 130 dev->data->nb_tx_queues, 131 &dev->data->dev_conf); 132 if (ret) { 133 if (!fs_err(sdev, ret)) 134 continue; 135 ERROR("Could not configure sub_device %d", i); 136 fs_unlock(dev, 0); 137 return ret; 138 } 139 if (rmv_interrupt && sdev->rmv_callback == 0) { 140 ret = rte_eth_dev_callback_register(PORT_ID(sdev), 141 RTE_ETH_EVENT_INTR_RMV, 142 failsafe_eth_rmv_event_callback, 143 sdev); 144 if (ret) 145 WARN("Failed to register RMV callback for sub_device %d", 146 SUB_ID(sdev)); 147 else 148 sdev->rmv_callback = 1; 149 } 150 dev->data->dev_conf.intr_conf.rmv = 0; 151 if (lsc_interrupt && sdev->lsc_callback == 0) { 152 ret = rte_eth_dev_callback_register(PORT_ID(sdev), 153 RTE_ETH_EVENT_INTR_LSC, 154 failsafe_eth_lsc_event_callback, 155 dev); 156 if (ret) 157 WARN("Failed to register LSC callback for sub_device %d", 158 SUB_ID(sdev)); 159 else 160 sdev->lsc_callback = 1; 161 } 162 dev->data->dev_conf.intr_conf.lsc = lsc_enabled; 163 sdev->state = DEV_ACTIVE; 164 } 165 if (PRIV(dev)->state < DEV_ACTIVE) 166 PRIV(dev)->state = DEV_ACTIVE; 167 fs_unlock(dev, 0); 168 return 0; 169 } 170 171 static int 172 fs_dev_start(struct rte_eth_dev *dev) 173 { 174 struct sub_device *sdev; 175 uint8_t i; 176 int ret; 177 178 fs_lock(dev, 0); 179 ret = failsafe_rx_intr_install(dev); 180 if (ret) { 181 fs_unlock(dev, 0); 182 return ret; 183 } 184 FOREACH_SUBDEV(sdev, i, dev) { 185 if (sdev->state != DEV_ACTIVE) 186 continue; 187 DEBUG("Starting sub_device %d", i); 188 ret = rte_eth_dev_start(PORT_ID(sdev)); 189 if (ret) { 190 if (!fs_err(sdev, ret)) 191 continue; 192 fs_unlock(dev, 0); 193 return ret; 194 } 195 ret = failsafe_rx_intr_install_subdevice(sdev); 196 if (ret) { 197 if (!fs_err(sdev, ret)) 198 continue; 199 rte_eth_dev_stop(PORT_ID(sdev)); 200 fs_unlock(dev, 0); 201 return ret; 202 } 203 sdev->state = DEV_STARTED; 204 } 205 if (PRIV(dev)->state < DEV_STARTED) 206 PRIV(dev)->state = DEV_STARTED; 207 fs_switch_dev(dev, NULL); 208 fs_unlock(dev, 0); 209 return 0; 210 } 211 212 static void 213 fs_dev_stop(struct rte_eth_dev *dev) 214 { 215 struct sub_device *sdev; 216 uint8_t i; 217 218 fs_lock(dev, 0); 219 PRIV(dev)->state = DEV_STARTED - 1; 220 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) { 221 rte_eth_dev_stop(PORT_ID(sdev)); 222 failsafe_rx_intr_uninstall_subdevice(sdev); 223 sdev->state = DEV_STARTED - 1; 224 } 225 failsafe_rx_intr_uninstall(dev); 226 fs_unlock(dev, 0); 227 } 228 229 static int 230 fs_dev_set_link_up(struct rte_eth_dev *dev) 231 { 232 struct sub_device *sdev; 233 uint8_t i; 234 int ret; 235 236 fs_lock(dev, 0); 237 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 238 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i); 239 ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); 240 if ((ret = fs_err(sdev, ret))) { 241 ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d" 242 " with error %d", i, ret); 243 fs_unlock(dev, 0); 244 return ret; 245 } 246 } 247 fs_unlock(dev, 0); 248 return 0; 249 } 250 251 static int 252 fs_dev_set_link_down(struct rte_eth_dev *dev) 253 { 254 struct sub_device *sdev; 255 uint8_t i; 256 int ret; 257 258 fs_lock(dev, 0); 259 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 260 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i); 261 ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); 262 if ((ret = fs_err(sdev, ret))) { 263 ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d" 264 " with error %d", i, ret); 265 fs_unlock(dev, 0); 266 return ret; 267 } 268 } 269 fs_unlock(dev, 0); 270 return 0; 271 } 272 273 static void fs_dev_free_queues(struct rte_eth_dev *dev); 274 static void 275 fs_dev_close(struct rte_eth_dev *dev) 276 { 277 struct sub_device *sdev; 278 uint8_t i; 279 280 fs_lock(dev, 0); 281 failsafe_hotplug_alarm_cancel(dev); 282 if (PRIV(dev)->state == DEV_STARTED) 283 dev->dev_ops->dev_stop(dev); 284 PRIV(dev)->state = DEV_ACTIVE - 1; 285 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 286 DEBUG("Closing sub_device %d", i); 287 failsafe_eth_dev_unregister_callbacks(sdev); 288 rte_eth_dev_close(PORT_ID(sdev)); 289 sdev->state = DEV_ACTIVE - 1; 290 } 291 fs_dev_free_queues(dev); 292 fs_unlock(dev, 0); 293 } 294 295 static void 296 fs_rx_queue_release(void *queue) 297 { 298 struct rte_eth_dev *dev; 299 struct sub_device *sdev; 300 uint8_t i; 301 struct rxq *rxq; 302 303 if (queue == NULL) 304 return; 305 rxq = queue; 306 dev = rxq->priv->dev; 307 fs_lock(dev, 0); 308 if (rxq->event_fd > 0) 309 close(rxq->event_fd); 310 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 311 SUBOPS(sdev, rx_queue_release) 312 (ETH(sdev)->data->rx_queues[rxq->qid]); 313 dev->data->rx_queues[rxq->qid] = NULL; 314 rte_free(rxq); 315 fs_unlock(dev, 0); 316 } 317 318 static int 319 fs_rx_queue_setup(struct rte_eth_dev *dev, 320 uint16_t rx_queue_id, 321 uint16_t nb_rx_desc, 322 unsigned int socket_id, 323 const struct rte_eth_rxconf *rx_conf, 324 struct rte_mempool *mb_pool) 325 { 326 /* 327 * FIXME: Add a proper interface in rte_eal_interrupts for 328 * allocating eventfd as an interrupt vector. 329 * For the time being, fake as if we are using MSIX interrupts, 330 * this will cause rte_intr_efd_enable to allocate an eventfd for us. 331 */ 332 struct rte_intr_handle intr_handle = { 333 .type = RTE_INTR_HANDLE_VFIO_MSIX, 334 .efds = { -1, }, 335 }; 336 struct sub_device *sdev; 337 struct rxq *rxq; 338 uint8_t i; 339 int ret; 340 341 fs_lock(dev, 0); 342 rxq = dev->data->rx_queues[rx_queue_id]; 343 if (rxq != NULL) { 344 fs_rx_queue_release(rxq); 345 dev->data->rx_queues[rx_queue_id] = NULL; 346 } 347 rxq = rte_zmalloc(NULL, 348 sizeof(*rxq) + 349 sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, 350 RTE_CACHE_LINE_SIZE); 351 if (rxq == NULL) { 352 fs_unlock(dev, 0); 353 return -ENOMEM; 354 } 355 FOREACH_SUBDEV(sdev, i, dev) 356 rte_atomic64_init(&rxq->refcnt[i]); 357 rxq->qid = rx_queue_id; 358 rxq->socket_id = socket_id; 359 rxq->info.mp = mb_pool; 360 rxq->info.conf = *rx_conf; 361 rxq->info.nb_desc = nb_rx_desc; 362 rxq->priv = PRIV(dev); 363 rxq->sdev = PRIV(dev)->subs; 364 ret = rte_intr_efd_enable(&intr_handle, 1); 365 if (ret < 0) { 366 fs_unlock(dev, 0); 367 return ret; 368 } 369 rxq->event_fd = intr_handle.efds[0]; 370 dev->data->rx_queues[rx_queue_id] = rxq; 371 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 372 ret = rte_eth_rx_queue_setup(PORT_ID(sdev), 373 rx_queue_id, 374 nb_rx_desc, socket_id, 375 rx_conf, mb_pool); 376 if ((ret = fs_err(sdev, ret))) { 377 ERROR("RX queue setup failed for sub_device %d", i); 378 goto free_rxq; 379 } 380 } 381 fs_unlock(dev, 0); 382 return 0; 383 free_rxq: 384 fs_rx_queue_release(rxq); 385 fs_unlock(dev, 0); 386 return ret; 387 } 388 389 static int 390 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) 391 { 392 struct rxq *rxq; 393 struct sub_device *sdev; 394 uint8_t i; 395 int ret; 396 int rc = 0; 397 398 fs_lock(dev, 0); 399 if (idx >= dev->data->nb_rx_queues) { 400 rc = -EINVAL; 401 goto unlock; 402 } 403 rxq = dev->data->rx_queues[idx]; 404 if (rxq == NULL || rxq->event_fd <= 0) { 405 rc = -EINVAL; 406 goto unlock; 407 } 408 /* Fail if proxy service is nor running. */ 409 if (PRIV(dev)->rxp.sstate != SS_RUNNING) { 410 ERROR("failsafe interrupt services are not running"); 411 rc = -EAGAIN; 412 goto unlock; 413 } 414 rxq->enable_events = 1; 415 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 416 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx); 417 ret = fs_err(sdev, ret); 418 if (ret) 419 rc = ret; 420 } 421 unlock: 422 fs_unlock(dev, 0); 423 if (rc) 424 rte_errno = -rc; 425 return rc; 426 } 427 428 static int 429 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx) 430 { 431 struct rxq *rxq; 432 struct sub_device *sdev; 433 uint64_t u64; 434 uint8_t i; 435 int rc = 0; 436 int ret; 437 438 fs_lock(dev, 0); 439 if (idx >= dev->data->nb_rx_queues) { 440 rc = -EINVAL; 441 goto unlock; 442 } 443 rxq = dev->data->rx_queues[idx]; 444 if (rxq == NULL || rxq->event_fd <= 0) { 445 rc = -EINVAL; 446 goto unlock; 447 } 448 rxq->enable_events = 0; 449 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 450 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx); 451 ret = fs_err(sdev, ret); 452 if (ret) 453 rc = ret; 454 } 455 /* Clear pending events */ 456 while (read(rxq->event_fd, &u64, sizeof(uint64_t)) > 0) 457 ; 458 unlock: 459 fs_unlock(dev, 0); 460 if (rc) 461 rte_errno = -rc; 462 return rc; 463 } 464 465 static void 466 fs_tx_queue_release(void *queue) 467 { 468 struct rte_eth_dev *dev; 469 struct sub_device *sdev; 470 uint8_t i; 471 struct txq *txq; 472 473 if (queue == NULL) 474 return; 475 txq = queue; 476 dev = txq->priv->dev; 477 fs_lock(dev, 0); 478 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 479 SUBOPS(sdev, tx_queue_release) 480 (ETH(sdev)->data->tx_queues[txq->qid]); 481 dev->data->tx_queues[txq->qid] = NULL; 482 rte_free(txq); 483 fs_unlock(dev, 0); 484 } 485 486 static int 487 fs_tx_queue_setup(struct rte_eth_dev *dev, 488 uint16_t tx_queue_id, 489 uint16_t nb_tx_desc, 490 unsigned int socket_id, 491 const struct rte_eth_txconf *tx_conf) 492 { 493 struct sub_device *sdev; 494 struct txq *txq; 495 uint8_t i; 496 int ret; 497 498 fs_lock(dev, 0); 499 txq = dev->data->tx_queues[tx_queue_id]; 500 if (txq != NULL) { 501 fs_tx_queue_release(txq); 502 dev->data->tx_queues[tx_queue_id] = NULL; 503 } 504 txq = rte_zmalloc("ethdev TX queue", 505 sizeof(*txq) + 506 sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, 507 RTE_CACHE_LINE_SIZE); 508 if (txq == NULL) { 509 fs_unlock(dev, 0); 510 return -ENOMEM; 511 } 512 FOREACH_SUBDEV(sdev, i, dev) 513 rte_atomic64_init(&txq->refcnt[i]); 514 txq->qid = tx_queue_id; 515 txq->socket_id = socket_id; 516 txq->info.conf = *tx_conf; 517 txq->info.nb_desc = nb_tx_desc; 518 txq->priv = PRIV(dev); 519 dev->data->tx_queues[tx_queue_id] = txq; 520 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 521 ret = rte_eth_tx_queue_setup(PORT_ID(sdev), 522 tx_queue_id, 523 nb_tx_desc, socket_id, 524 tx_conf); 525 if ((ret = fs_err(sdev, ret))) { 526 ERROR("TX queue setup failed for sub_device %d", i); 527 goto free_txq; 528 } 529 } 530 fs_unlock(dev, 0); 531 return 0; 532 free_txq: 533 fs_tx_queue_release(txq); 534 fs_unlock(dev, 0); 535 return ret; 536 } 537 538 static void 539 fs_dev_free_queues(struct rte_eth_dev *dev) 540 { 541 uint16_t i; 542 543 for (i = 0; i < dev->data->nb_rx_queues; i++) { 544 fs_rx_queue_release(dev->data->rx_queues[i]); 545 dev->data->rx_queues[i] = NULL; 546 } 547 dev->data->nb_rx_queues = 0; 548 for (i = 0; i < dev->data->nb_tx_queues; i++) { 549 fs_tx_queue_release(dev->data->tx_queues[i]); 550 dev->data->tx_queues[i] = NULL; 551 } 552 dev->data->nb_tx_queues = 0; 553 } 554 555 static void 556 fs_promiscuous_enable(struct rte_eth_dev *dev) 557 { 558 struct sub_device *sdev; 559 uint8_t i; 560 561 fs_lock(dev, 0); 562 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 563 rte_eth_promiscuous_enable(PORT_ID(sdev)); 564 fs_unlock(dev, 0); 565 } 566 567 static void 568 fs_promiscuous_disable(struct rte_eth_dev *dev) 569 { 570 struct sub_device *sdev; 571 uint8_t i; 572 573 fs_lock(dev, 0); 574 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 575 rte_eth_promiscuous_disable(PORT_ID(sdev)); 576 fs_unlock(dev, 0); 577 } 578 579 static void 580 fs_allmulticast_enable(struct rte_eth_dev *dev) 581 { 582 struct sub_device *sdev; 583 uint8_t i; 584 585 fs_lock(dev, 0); 586 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 587 rte_eth_allmulticast_enable(PORT_ID(sdev)); 588 fs_unlock(dev, 0); 589 } 590 591 static void 592 fs_allmulticast_disable(struct rte_eth_dev *dev) 593 { 594 struct sub_device *sdev; 595 uint8_t i; 596 597 fs_lock(dev, 0); 598 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 599 rte_eth_allmulticast_disable(PORT_ID(sdev)); 600 fs_unlock(dev, 0); 601 } 602 603 static int 604 fs_link_update(struct rte_eth_dev *dev, 605 int wait_to_complete) 606 { 607 struct sub_device *sdev; 608 uint8_t i; 609 int ret; 610 611 fs_lock(dev, 0); 612 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 613 DEBUG("Calling link_update on sub_device %d", i); 614 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete); 615 if (ret && ret != -1 && sdev->remove == 0 && 616 rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) { 617 ERROR("Link update failed for sub_device %d with error %d", 618 i, ret); 619 fs_unlock(dev, 0); 620 return ret; 621 } 622 } 623 if (TX_SUBDEV(dev)) { 624 struct rte_eth_link *l1; 625 struct rte_eth_link *l2; 626 627 l1 = &dev->data->dev_link; 628 l2 = Ð(TX_SUBDEV(dev))->data->dev_link; 629 if (memcmp(l1, l2, sizeof(*l1))) { 630 *l1 = *l2; 631 fs_unlock(dev, 0); 632 return 0; 633 } 634 } 635 fs_unlock(dev, 0); 636 return -1; 637 } 638 639 static int 640 fs_stats_get(struct rte_eth_dev *dev, 641 struct rte_eth_stats *stats) 642 { 643 struct rte_eth_stats backup; 644 struct sub_device *sdev; 645 uint8_t i; 646 int ret; 647 648 fs_lock(dev, 0); 649 rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats)); 650 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 651 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats; 652 uint64_t *timestamp = &sdev->stats_snapshot.timestamp; 653 654 rte_memcpy(&backup, snapshot, sizeof(backup)); 655 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot); 656 if (ret) { 657 if (!fs_err(sdev, ret)) { 658 rte_memcpy(snapshot, &backup, sizeof(backup)); 659 goto inc; 660 } 661 ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d", 662 i, ret); 663 *timestamp = 0; 664 fs_unlock(dev, 0); 665 return ret; 666 } 667 *timestamp = rte_rdtsc(); 668 inc: 669 failsafe_stats_increment(stats, snapshot); 670 } 671 fs_unlock(dev, 0); 672 return 0; 673 } 674 675 static void 676 fs_stats_reset(struct rte_eth_dev *dev) 677 { 678 struct sub_device *sdev; 679 uint8_t i; 680 681 fs_lock(dev, 0); 682 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 683 rte_eth_stats_reset(PORT_ID(sdev)); 684 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats)); 685 } 686 memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats)); 687 fs_unlock(dev, 0); 688 } 689 690 /** 691 * Fail-safe dev_infos_get rules: 692 * 693 * No sub_device: 694 * Numerables: 695 * Use the maximum possible values for any field, so as not 696 * to impede any further configuration effort. 697 * Capabilities: 698 * Limits capabilities to those that are understood by the 699 * fail-safe PMD. This understanding stems from the fail-safe 700 * being capable of verifying that the related capability is 701 * expressed within the device configuration (struct rte_eth_conf). 702 * 703 * At least one probed sub_device: 704 * Numerables: 705 * Uses values from the active probed sub_device 706 * The rationale here is that if any sub_device is less capable 707 * (for example concerning the number of queues) than the active 708 * sub_device, then its subsequent configuration will fail. 709 * It is impossible to foresee this failure when the failing sub_device 710 * is supposed to be plugged-in later on, so the configuration process 711 * is the single point of failure and error reporting. 712 * Capabilities: 713 * Uses a logical AND of RX capabilities among 714 * all sub_devices and the default capabilities. 715 * Uses a logical AND of TX capabilities among 716 * the active probed sub_device and the default capabilities. 717 * 718 */ 719 static void 720 fs_dev_infos_get(struct rte_eth_dev *dev, 721 struct rte_eth_dev_info *infos) 722 { 723 struct sub_device *sdev; 724 uint8_t i; 725 726 sdev = TX_SUBDEV(dev); 727 if (sdev == NULL) { 728 DEBUG("No probed device, using default infos"); 729 rte_memcpy(&PRIV(dev)->infos, &default_infos, 730 sizeof(default_infos)); 731 } else { 732 uint64_t rx_offload_capa; 733 uint64_t rxq_offload_capa; 734 uint64_t rss_hf_offload_capa; 735 736 rx_offload_capa = default_infos.rx_offload_capa; 737 rxq_offload_capa = default_infos.rx_queue_offload_capa; 738 rss_hf_offload_capa = default_infos.flow_type_rss_offloads; 739 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { 740 rte_eth_dev_info_get(PORT_ID(sdev), 741 &PRIV(dev)->infos); 742 rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa; 743 rxq_offload_capa &= 744 PRIV(dev)->infos.rx_queue_offload_capa; 745 rss_hf_offload_capa &= 746 PRIV(dev)->infos.flow_type_rss_offloads; 747 } 748 sdev = TX_SUBDEV(dev); 749 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos); 750 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa; 751 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa; 752 PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa; 753 PRIV(dev)->infos.tx_offload_capa &= 754 default_infos.tx_offload_capa; 755 PRIV(dev)->infos.tx_queue_offload_capa &= 756 default_infos.tx_queue_offload_capa; 757 } 758 rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos)); 759 } 760 761 static const uint32_t * 762 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev) 763 { 764 struct sub_device *sdev; 765 struct rte_eth_dev *edev; 766 const uint32_t *ret; 767 768 fs_lock(dev, 0); 769 sdev = TX_SUBDEV(dev); 770 if (sdev == NULL) { 771 ret = NULL; 772 goto unlock; 773 } 774 edev = ETH(sdev); 775 /* ENOTSUP: counts as no supported ptypes */ 776 if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) { 777 ret = NULL; 778 goto unlock; 779 } 780 /* 781 * The API does not permit to do a clean AND of all ptypes, 782 * It is also incomplete by design and we do not really care 783 * to have a best possible value in this context. 784 * We just return the ptypes of the device of highest 785 * priority, usually the PREFERRED device. 786 */ 787 ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev); 788 unlock: 789 fs_unlock(dev, 0); 790 return ret; 791 } 792 793 static int 794 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 795 { 796 struct sub_device *sdev; 797 uint8_t i; 798 int ret; 799 800 fs_lock(dev, 0); 801 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 802 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i); 803 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu); 804 if ((ret = fs_err(sdev, ret))) { 805 ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d", 806 i, ret); 807 fs_unlock(dev, 0); 808 return ret; 809 } 810 } 811 fs_unlock(dev, 0); 812 return 0; 813 } 814 815 static int 816 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 817 { 818 struct sub_device *sdev; 819 uint8_t i; 820 int ret; 821 822 fs_lock(dev, 0); 823 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 824 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i); 825 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on); 826 if ((ret = fs_err(sdev, ret))) { 827 ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d" 828 " with error %d", i, ret); 829 fs_unlock(dev, 0); 830 return ret; 831 } 832 } 833 fs_unlock(dev, 0); 834 return 0; 835 } 836 837 static int 838 fs_flow_ctrl_get(struct rte_eth_dev *dev, 839 struct rte_eth_fc_conf *fc_conf) 840 { 841 struct sub_device *sdev; 842 int ret; 843 844 fs_lock(dev, 0); 845 sdev = TX_SUBDEV(dev); 846 if (sdev == NULL) { 847 ret = 0; 848 goto unlock; 849 } 850 if (SUBOPS(sdev, flow_ctrl_get) == NULL) { 851 ret = -ENOTSUP; 852 goto unlock; 853 } 854 ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf); 855 unlock: 856 fs_unlock(dev, 0); 857 return ret; 858 } 859 860 static int 861 fs_flow_ctrl_set(struct rte_eth_dev *dev, 862 struct rte_eth_fc_conf *fc_conf) 863 { 864 struct sub_device *sdev; 865 uint8_t i; 866 int ret; 867 868 fs_lock(dev, 0); 869 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 870 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i); 871 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf); 872 if ((ret = fs_err(sdev, ret))) { 873 ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d" 874 " with error %d", i, ret); 875 fs_unlock(dev, 0); 876 return ret; 877 } 878 } 879 fs_unlock(dev, 0); 880 return 0; 881 } 882 883 static void 884 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 885 { 886 struct sub_device *sdev; 887 uint8_t i; 888 889 fs_lock(dev, 0); 890 /* No check: already done within the rte_eth_dev_mac_addr_remove 891 * call for the fail-safe device. 892 */ 893 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 894 rte_eth_dev_mac_addr_remove(PORT_ID(sdev), 895 &dev->data->mac_addrs[index]); 896 PRIV(dev)->mac_addr_pool[index] = 0; 897 fs_unlock(dev, 0); 898 } 899 900 static int 901 fs_mac_addr_add(struct rte_eth_dev *dev, 902 struct ether_addr *mac_addr, 903 uint32_t index, 904 uint32_t vmdq) 905 { 906 struct sub_device *sdev; 907 int ret; 908 uint8_t i; 909 910 RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR); 911 fs_lock(dev, 0); 912 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 913 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq); 914 if ((ret = fs_err(sdev, ret))) { 915 ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %" 916 PRIu8 " with error %d", i, ret); 917 fs_unlock(dev, 0); 918 return ret; 919 } 920 } 921 if (index >= PRIV(dev)->nb_mac_addr) { 922 DEBUG("Growing mac_addrs array"); 923 PRIV(dev)->nb_mac_addr = index; 924 } 925 PRIV(dev)->mac_addr_pool[index] = vmdq; 926 fs_unlock(dev, 0); 927 return 0; 928 } 929 930 static int 931 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 932 { 933 struct sub_device *sdev; 934 uint8_t i; 935 int ret; 936 937 fs_lock(dev, 0); 938 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 939 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr); 940 ret = fs_err(sdev, ret); 941 if (ret) { 942 ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d", 943 i, ret); 944 fs_unlock(dev, 0); 945 return ret; 946 } 947 } 948 fs_unlock(dev, 0); 949 950 return 0; 951 } 952 953 static int 954 fs_rss_hash_update(struct rte_eth_dev *dev, 955 struct rte_eth_rss_conf *rss_conf) 956 { 957 struct sub_device *sdev; 958 uint8_t i; 959 int ret; 960 961 fs_lock(dev, 0); 962 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 963 ret = rte_eth_dev_rss_hash_update(PORT_ID(sdev), rss_conf); 964 ret = fs_err(sdev, ret); 965 if (ret) { 966 ERROR("Operation rte_eth_dev_rss_hash_update" 967 " failed for sub_device %d with error %d", 968 i, ret); 969 fs_unlock(dev, 0); 970 return ret; 971 } 972 } 973 fs_unlock(dev, 0); 974 975 return 0; 976 } 977 978 static int 979 fs_filter_ctrl(struct rte_eth_dev *dev, 980 enum rte_filter_type type, 981 enum rte_filter_op op, 982 void *arg) 983 { 984 struct sub_device *sdev; 985 uint8_t i; 986 int ret; 987 988 if (type == RTE_ETH_FILTER_GENERIC && 989 op == RTE_ETH_FILTER_GET) { 990 *(const void **)arg = &fs_flow_ops; 991 return 0; 992 } 993 fs_lock(dev, 0); 994 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 995 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i); 996 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg); 997 if ((ret = fs_err(sdev, ret))) { 998 ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d" 999 " with error %d", i, ret); 1000 fs_unlock(dev, 0); 1001 return ret; 1002 } 1003 } 1004 fs_unlock(dev, 0); 1005 return 0; 1006 } 1007 1008 const struct eth_dev_ops failsafe_ops = { 1009 .dev_configure = fs_dev_configure, 1010 .dev_start = fs_dev_start, 1011 .dev_stop = fs_dev_stop, 1012 .dev_set_link_down = fs_dev_set_link_down, 1013 .dev_set_link_up = fs_dev_set_link_up, 1014 .dev_close = fs_dev_close, 1015 .promiscuous_enable = fs_promiscuous_enable, 1016 .promiscuous_disable = fs_promiscuous_disable, 1017 .allmulticast_enable = fs_allmulticast_enable, 1018 .allmulticast_disable = fs_allmulticast_disable, 1019 .link_update = fs_link_update, 1020 .stats_get = fs_stats_get, 1021 .stats_reset = fs_stats_reset, 1022 .dev_infos_get = fs_dev_infos_get, 1023 .dev_supported_ptypes_get = fs_dev_supported_ptypes_get, 1024 .mtu_set = fs_mtu_set, 1025 .vlan_filter_set = fs_vlan_filter_set, 1026 .rx_queue_setup = fs_rx_queue_setup, 1027 .tx_queue_setup = fs_tx_queue_setup, 1028 .rx_queue_release = fs_rx_queue_release, 1029 .tx_queue_release = fs_tx_queue_release, 1030 .rx_queue_intr_enable = fs_rx_intr_enable, 1031 .rx_queue_intr_disable = fs_rx_intr_disable, 1032 .flow_ctrl_get = fs_flow_ctrl_get, 1033 .flow_ctrl_set = fs_flow_ctrl_set, 1034 .mac_addr_remove = fs_mac_addr_remove, 1035 .mac_addr_add = fs_mac_addr_add, 1036 .mac_addr_set = fs_mac_addr_set, 1037 .rss_hash_update = fs_rss_hash_update, 1038 .filter_ctrl = fs_filter_ctrl, 1039 }; 1040