1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 #include <stdbool.h> 7 #include <stdint.h> 8 #include <unistd.h> 9 10 #include <rte_debug.h> 11 #include <rte_atomic.h> 12 #include <rte_ethdev_driver.h> 13 #include <rte_malloc.h> 14 #include <rte_flow.h> 15 #include <rte_cycles.h> 16 17 #include "failsafe_private.h" 18 19 static struct rte_eth_dev_info default_infos = { 20 /* Max possible number of elements */ 21 .max_rx_pktlen = UINT32_MAX, 22 .max_rx_queues = RTE_MAX_QUEUES_PER_PORT, 23 .max_tx_queues = RTE_MAX_QUEUES_PER_PORT, 24 .max_mac_addrs = FAILSAFE_MAX_ETHADDR, 25 .max_hash_mac_addrs = UINT32_MAX, 26 .max_vfs = UINT16_MAX, 27 .max_vmdq_pools = UINT16_MAX, 28 .rx_desc_lim = { 29 .nb_max = UINT16_MAX, 30 .nb_min = 0, 31 .nb_align = 1, 32 .nb_seg_max = UINT16_MAX, 33 .nb_mtu_seg_max = UINT16_MAX, 34 }, 35 .tx_desc_lim = { 36 .nb_max = UINT16_MAX, 37 .nb_min = 0, 38 .nb_align = 1, 39 .nb_seg_max = UINT16_MAX, 40 .nb_mtu_seg_max = UINT16_MAX, 41 }, 42 /* 43 * Set of capabilities that can be verified upon 44 * configuring a sub-device. 45 */ 46 .rx_offload_capa = 47 DEV_RX_OFFLOAD_VLAN_STRIP | 48 DEV_RX_OFFLOAD_IPV4_CKSUM | 49 DEV_RX_OFFLOAD_UDP_CKSUM | 50 DEV_RX_OFFLOAD_TCP_CKSUM | 51 DEV_RX_OFFLOAD_TCP_LRO | 52 DEV_RX_OFFLOAD_QINQ_STRIP | 53 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 54 DEV_RX_OFFLOAD_MACSEC_STRIP | 55 DEV_RX_OFFLOAD_HEADER_SPLIT | 56 DEV_RX_OFFLOAD_VLAN_FILTER | 57 DEV_RX_OFFLOAD_VLAN_EXTEND | 58 DEV_RX_OFFLOAD_JUMBO_FRAME | 59 DEV_RX_OFFLOAD_CRC_STRIP | 60 DEV_RX_OFFLOAD_SCATTER | 61 DEV_RX_OFFLOAD_TIMESTAMP | 62 DEV_RX_OFFLOAD_SECURITY, 63 .rx_queue_offload_capa = 64 DEV_RX_OFFLOAD_VLAN_STRIP | 65 DEV_RX_OFFLOAD_IPV4_CKSUM | 66 DEV_RX_OFFLOAD_UDP_CKSUM | 67 DEV_RX_OFFLOAD_TCP_CKSUM | 68 DEV_RX_OFFLOAD_TCP_LRO | 69 DEV_RX_OFFLOAD_QINQ_STRIP | 70 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 71 DEV_RX_OFFLOAD_MACSEC_STRIP | 72 DEV_RX_OFFLOAD_HEADER_SPLIT | 73 DEV_RX_OFFLOAD_VLAN_FILTER | 74 DEV_RX_OFFLOAD_VLAN_EXTEND | 75 DEV_RX_OFFLOAD_JUMBO_FRAME | 76 DEV_RX_OFFLOAD_CRC_STRIP | 77 DEV_RX_OFFLOAD_SCATTER | 78 DEV_RX_OFFLOAD_TIMESTAMP | 79 DEV_RX_OFFLOAD_SECURITY, 80 .tx_offload_capa = 81 DEV_TX_OFFLOAD_MULTI_SEGS | 82 DEV_TX_OFFLOAD_IPV4_CKSUM | 83 DEV_TX_OFFLOAD_UDP_CKSUM | 84 DEV_TX_OFFLOAD_TCP_CKSUM | 85 DEV_TX_OFFLOAD_TCP_TSO, 86 .flow_type_rss_offloads = 87 ETH_RSS_IP | 88 ETH_RSS_UDP | 89 ETH_RSS_TCP, 90 }; 91 92 static int 93 fs_dev_configure(struct rte_eth_dev *dev) 94 { 95 struct sub_device *sdev; 96 uint8_t i; 97 int ret; 98 99 fs_lock(dev, 0); 100 FOREACH_SUBDEV(sdev, i, dev) { 101 int rmv_interrupt = 0; 102 int lsc_interrupt = 0; 103 int lsc_enabled; 104 105 if (sdev->state != DEV_PROBED && 106 !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE)) 107 continue; 108 109 rmv_interrupt = ETH(sdev)->data->dev_flags & 110 RTE_ETH_DEV_INTR_RMV; 111 if (rmv_interrupt) { 112 DEBUG("Enabling RMV interrupts for sub_device %d", i); 113 dev->data->dev_conf.intr_conf.rmv = 1; 114 } else { 115 DEBUG("sub_device %d does not support RMV event", i); 116 } 117 lsc_enabled = dev->data->dev_conf.intr_conf.lsc; 118 lsc_interrupt = lsc_enabled && 119 (ETH(sdev)->data->dev_flags & 120 RTE_ETH_DEV_INTR_LSC); 121 if (lsc_interrupt) { 122 DEBUG("Enabling LSC interrupts for sub_device %d", i); 123 dev->data->dev_conf.intr_conf.lsc = 1; 124 } else if (lsc_enabled && !lsc_interrupt) { 125 DEBUG("Disabling LSC interrupts for sub_device %d", i); 126 dev->data->dev_conf.intr_conf.lsc = 0; 127 } 128 DEBUG("Configuring sub-device %d", i); 129 ret = rte_eth_dev_configure(PORT_ID(sdev), 130 dev->data->nb_rx_queues, 131 dev->data->nb_tx_queues, 132 &dev->data->dev_conf); 133 if (ret) { 134 if (!fs_err(sdev, ret)) 135 continue; 136 ERROR("Could not configure sub_device %d", i); 137 fs_unlock(dev, 0); 138 return ret; 139 } 140 if (rmv_interrupt) { 141 ret = rte_eth_dev_callback_register(PORT_ID(sdev), 142 RTE_ETH_EVENT_INTR_RMV, 143 failsafe_eth_rmv_event_callback, 144 sdev); 145 if (ret) 146 WARN("Failed to register RMV callback for sub_device %d", 147 SUB_ID(sdev)); 148 } 149 dev->data->dev_conf.intr_conf.rmv = 0; 150 if (lsc_interrupt) { 151 ret = rte_eth_dev_callback_register(PORT_ID(sdev), 152 RTE_ETH_EVENT_INTR_LSC, 153 failsafe_eth_lsc_event_callback, 154 dev); 155 if (ret) 156 WARN("Failed to register LSC callback for sub_device %d", 157 SUB_ID(sdev)); 158 } 159 dev->data->dev_conf.intr_conf.lsc = lsc_enabled; 160 sdev->state = DEV_ACTIVE; 161 } 162 if (PRIV(dev)->state < DEV_ACTIVE) 163 PRIV(dev)->state = DEV_ACTIVE; 164 fs_unlock(dev, 0); 165 return 0; 166 } 167 168 static int 169 fs_dev_start(struct rte_eth_dev *dev) 170 { 171 struct sub_device *sdev; 172 uint8_t i; 173 int ret; 174 175 fs_lock(dev, 0); 176 ret = failsafe_rx_intr_install(dev); 177 if (ret) { 178 fs_unlock(dev, 0); 179 return ret; 180 } 181 FOREACH_SUBDEV(sdev, i, dev) { 182 if (sdev->state != DEV_ACTIVE) 183 continue; 184 DEBUG("Starting sub_device %d", i); 185 ret = rte_eth_dev_start(PORT_ID(sdev)); 186 if (ret) { 187 if (!fs_err(sdev, ret)) 188 continue; 189 fs_unlock(dev, 0); 190 return ret; 191 } 192 ret = failsafe_rx_intr_install_subdevice(sdev); 193 if (ret) { 194 if (!fs_err(sdev, ret)) 195 continue; 196 rte_eth_dev_stop(PORT_ID(sdev)); 197 fs_unlock(dev, 0); 198 return ret; 199 } 200 sdev->state = DEV_STARTED; 201 } 202 if (PRIV(dev)->state < DEV_STARTED) 203 PRIV(dev)->state = DEV_STARTED; 204 fs_switch_dev(dev, NULL); 205 fs_unlock(dev, 0); 206 return 0; 207 } 208 209 static void 210 fs_dev_stop(struct rte_eth_dev *dev) 211 { 212 struct sub_device *sdev; 213 uint8_t i; 214 215 fs_lock(dev, 0); 216 PRIV(dev)->state = DEV_STARTED - 1; 217 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) { 218 rte_eth_dev_stop(PORT_ID(sdev)); 219 failsafe_rx_intr_uninstall_subdevice(sdev); 220 sdev->state = DEV_STARTED - 1; 221 } 222 failsafe_rx_intr_uninstall(dev); 223 fs_unlock(dev, 0); 224 } 225 226 static int 227 fs_dev_set_link_up(struct rte_eth_dev *dev) 228 { 229 struct sub_device *sdev; 230 uint8_t i; 231 int ret; 232 233 fs_lock(dev, 0); 234 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 235 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i); 236 ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); 237 if ((ret = fs_err(sdev, ret))) { 238 ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d" 239 " with error %d", i, ret); 240 fs_unlock(dev, 0); 241 return ret; 242 } 243 } 244 fs_unlock(dev, 0); 245 return 0; 246 } 247 248 static int 249 fs_dev_set_link_down(struct rte_eth_dev *dev) 250 { 251 struct sub_device *sdev; 252 uint8_t i; 253 int ret; 254 255 fs_lock(dev, 0); 256 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 257 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i); 258 ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); 259 if ((ret = fs_err(sdev, ret))) { 260 ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d" 261 " with error %d", i, ret); 262 fs_unlock(dev, 0); 263 return ret; 264 } 265 } 266 fs_unlock(dev, 0); 267 return 0; 268 } 269 270 static void fs_dev_free_queues(struct rte_eth_dev *dev); 271 static void 272 fs_dev_close(struct rte_eth_dev *dev) 273 { 274 struct sub_device *sdev; 275 uint8_t i; 276 277 fs_lock(dev, 0); 278 failsafe_hotplug_alarm_cancel(dev); 279 if (PRIV(dev)->state == DEV_STARTED) 280 dev->dev_ops->dev_stop(dev); 281 PRIV(dev)->state = DEV_ACTIVE - 1; 282 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 283 DEBUG("Closing sub_device %d", i); 284 rte_eth_dev_close(PORT_ID(sdev)); 285 sdev->state = DEV_ACTIVE - 1; 286 } 287 fs_dev_free_queues(dev); 288 fs_unlock(dev, 0); 289 } 290 291 static void 292 fs_rx_queue_release(void *queue) 293 { 294 struct rte_eth_dev *dev; 295 struct sub_device *sdev; 296 uint8_t i; 297 struct rxq *rxq; 298 299 if (queue == NULL) 300 return; 301 rxq = queue; 302 dev = rxq->priv->dev; 303 fs_lock(dev, 0); 304 if (rxq->event_fd > 0) 305 close(rxq->event_fd); 306 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 307 SUBOPS(sdev, rx_queue_release) 308 (ETH(sdev)->data->rx_queues[rxq->qid]); 309 dev->data->rx_queues[rxq->qid] = NULL; 310 rte_free(rxq); 311 fs_unlock(dev, 0); 312 } 313 314 static int 315 fs_rx_queue_setup(struct rte_eth_dev *dev, 316 uint16_t rx_queue_id, 317 uint16_t nb_rx_desc, 318 unsigned int socket_id, 319 const struct rte_eth_rxconf *rx_conf, 320 struct rte_mempool *mb_pool) 321 { 322 /* 323 * FIXME: Add a proper interface in rte_eal_interrupts for 324 * allocating eventfd as an interrupt vector. 325 * For the time being, fake as if we are using MSIX interrupts, 326 * this will cause rte_intr_efd_enable to allocate an eventfd for us. 327 */ 328 struct rte_intr_handle intr_handle = { 329 .type = RTE_INTR_HANDLE_VFIO_MSIX, 330 .efds = { -1, }, 331 }; 332 struct sub_device *sdev; 333 struct rxq *rxq; 334 uint8_t i; 335 int ret; 336 337 fs_lock(dev, 0); 338 rxq = dev->data->rx_queues[rx_queue_id]; 339 if (rxq != NULL) { 340 fs_rx_queue_release(rxq); 341 dev->data->rx_queues[rx_queue_id] = NULL; 342 } 343 rxq = rte_zmalloc(NULL, 344 sizeof(*rxq) + 345 sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, 346 RTE_CACHE_LINE_SIZE); 347 if (rxq == NULL) { 348 fs_unlock(dev, 0); 349 return -ENOMEM; 350 } 351 FOREACH_SUBDEV(sdev, i, dev) 352 rte_atomic64_init(&rxq->refcnt[i]); 353 rxq->qid = rx_queue_id; 354 rxq->socket_id = socket_id; 355 rxq->info.mp = mb_pool; 356 rxq->info.conf = *rx_conf; 357 rxq->info.nb_desc = nb_rx_desc; 358 rxq->priv = PRIV(dev); 359 rxq->sdev = PRIV(dev)->subs; 360 ret = rte_intr_efd_enable(&intr_handle, 1); 361 if (ret < 0) { 362 fs_unlock(dev, 0); 363 return ret; 364 } 365 rxq->event_fd = intr_handle.efds[0]; 366 dev->data->rx_queues[rx_queue_id] = rxq; 367 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 368 ret = rte_eth_rx_queue_setup(PORT_ID(sdev), 369 rx_queue_id, 370 nb_rx_desc, socket_id, 371 rx_conf, mb_pool); 372 if ((ret = fs_err(sdev, ret))) { 373 ERROR("RX queue setup failed for sub_device %d", i); 374 goto free_rxq; 375 } 376 } 377 fs_unlock(dev, 0); 378 return 0; 379 free_rxq: 380 fs_rx_queue_release(rxq); 381 fs_unlock(dev, 0); 382 return ret; 383 } 384 385 static int 386 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) 387 { 388 struct rxq *rxq; 389 struct sub_device *sdev; 390 uint8_t i; 391 int ret; 392 int rc = 0; 393 394 fs_lock(dev, 0); 395 if (idx >= dev->data->nb_rx_queues) { 396 rc = -EINVAL; 397 goto unlock; 398 } 399 rxq = dev->data->rx_queues[idx]; 400 if (rxq == NULL || rxq->event_fd <= 0) { 401 rc = -EINVAL; 402 goto unlock; 403 } 404 /* Fail if proxy service is nor running. */ 405 if (PRIV(dev)->rxp.sstate != SS_RUNNING) { 406 ERROR("failsafe interrupt services are not running"); 407 rc = -EAGAIN; 408 goto unlock; 409 } 410 rxq->enable_events = 1; 411 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 412 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx); 413 ret = fs_err(sdev, ret); 414 if (ret) 415 rc = ret; 416 } 417 unlock: 418 fs_unlock(dev, 0); 419 if (rc) 420 rte_errno = -rc; 421 return rc; 422 } 423 424 static int 425 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx) 426 { 427 struct rxq *rxq; 428 struct sub_device *sdev; 429 uint64_t u64; 430 uint8_t i; 431 int rc = 0; 432 int ret; 433 434 fs_lock(dev, 0); 435 if (idx >= dev->data->nb_rx_queues) { 436 rc = -EINVAL; 437 goto unlock; 438 } 439 rxq = dev->data->rx_queues[idx]; 440 if (rxq == NULL || rxq->event_fd <= 0) { 441 rc = -EINVAL; 442 goto unlock; 443 } 444 rxq->enable_events = 0; 445 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 446 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx); 447 ret = fs_err(sdev, ret); 448 if (ret) 449 rc = ret; 450 } 451 /* Clear pending events */ 452 while (read(rxq->event_fd, &u64, sizeof(uint64_t)) > 0) 453 ; 454 unlock: 455 fs_unlock(dev, 0); 456 if (rc) 457 rte_errno = -rc; 458 return rc; 459 } 460 461 static void 462 fs_tx_queue_release(void *queue) 463 { 464 struct rte_eth_dev *dev; 465 struct sub_device *sdev; 466 uint8_t i; 467 struct txq *txq; 468 469 if (queue == NULL) 470 return; 471 txq = queue; 472 dev = txq->priv->dev; 473 fs_lock(dev, 0); 474 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 475 SUBOPS(sdev, tx_queue_release) 476 (ETH(sdev)->data->tx_queues[txq->qid]); 477 dev->data->tx_queues[txq->qid] = NULL; 478 rte_free(txq); 479 fs_unlock(dev, 0); 480 } 481 482 static int 483 fs_tx_queue_setup(struct rte_eth_dev *dev, 484 uint16_t tx_queue_id, 485 uint16_t nb_tx_desc, 486 unsigned int socket_id, 487 const struct rte_eth_txconf *tx_conf) 488 { 489 struct sub_device *sdev; 490 struct txq *txq; 491 uint8_t i; 492 int ret; 493 494 fs_lock(dev, 0); 495 txq = dev->data->tx_queues[tx_queue_id]; 496 if (txq != NULL) { 497 fs_tx_queue_release(txq); 498 dev->data->tx_queues[tx_queue_id] = NULL; 499 } 500 txq = rte_zmalloc("ethdev TX queue", 501 sizeof(*txq) + 502 sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, 503 RTE_CACHE_LINE_SIZE); 504 if (txq == NULL) { 505 fs_unlock(dev, 0); 506 return -ENOMEM; 507 } 508 FOREACH_SUBDEV(sdev, i, dev) 509 rte_atomic64_init(&txq->refcnt[i]); 510 txq->qid = tx_queue_id; 511 txq->socket_id = socket_id; 512 txq->info.conf = *tx_conf; 513 txq->info.nb_desc = nb_tx_desc; 514 txq->priv = PRIV(dev); 515 dev->data->tx_queues[tx_queue_id] = txq; 516 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 517 ret = rte_eth_tx_queue_setup(PORT_ID(sdev), 518 tx_queue_id, 519 nb_tx_desc, socket_id, 520 tx_conf); 521 if ((ret = fs_err(sdev, ret))) { 522 ERROR("TX queue setup failed for sub_device %d", i); 523 goto free_txq; 524 } 525 } 526 fs_unlock(dev, 0); 527 return 0; 528 free_txq: 529 fs_tx_queue_release(txq); 530 fs_unlock(dev, 0); 531 return ret; 532 } 533 534 static void 535 fs_dev_free_queues(struct rte_eth_dev *dev) 536 { 537 uint16_t i; 538 539 for (i = 0; i < dev->data->nb_rx_queues; i++) { 540 fs_rx_queue_release(dev->data->rx_queues[i]); 541 dev->data->rx_queues[i] = NULL; 542 } 543 dev->data->nb_rx_queues = 0; 544 for (i = 0; i < dev->data->nb_tx_queues; i++) { 545 fs_tx_queue_release(dev->data->tx_queues[i]); 546 dev->data->tx_queues[i] = NULL; 547 } 548 dev->data->nb_tx_queues = 0; 549 } 550 551 static void 552 fs_promiscuous_enable(struct rte_eth_dev *dev) 553 { 554 struct sub_device *sdev; 555 uint8_t i; 556 557 fs_lock(dev, 0); 558 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 559 rte_eth_promiscuous_enable(PORT_ID(sdev)); 560 fs_unlock(dev, 0); 561 } 562 563 static void 564 fs_promiscuous_disable(struct rte_eth_dev *dev) 565 { 566 struct sub_device *sdev; 567 uint8_t i; 568 569 fs_lock(dev, 0); 570 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 571 rte_eth_promiscuous_disable(PORT_ID(sdev)); 572 fs_unlock(dev, 0); 573 } 574 575 static void 576 fs_allmulticast_enable(struct rte_eth_dev *dev) 577 { 578 struct sub_device *sdev; 579 uint8_t i; 580 581 fs_lock(dev, 0); 582 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 583 rte_eth_allmulticast_enable(PORT_ID(sdev)); 584 fs_unlock(dev, 0); 585 } 586 587 static void 588 fs_allmulticast_disable(struct rte_eth_dev *dev) 589 { 590 struct sub_device *sdev; 591 uint8_t i; 592 593 fs_lock(dev, 0); 594 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 595 rte_eth_allmulticast_disable(PORT_ID(sdev)); 596 fs_unlock(dev, 0); 597 } 598 599 static int 600 fs_link_update(struct rte_eth_dev *dev, 601 int wait_to_complete) 602 { 603 struct sub_device *sdev; 604 uint8_t i; 605 int ret; 606 607 fs_lock(dev, 0); 608 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 609 DEBUG("Calling link_update on sub_device %d", i); 610 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete); 611 if (ret && ret != -1 && sdev->remove == 0 && 612 rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) { 613 ERROR("Link update failed for sub_device %d with error %d", 614 i, ret); 615 fs_unlock(dev, 0); 616 return ret; 617 } 618 } 619 if (TX_SUBDEV(dev)) { 620 struct rte_eth_link *l1; 621 struct rte_eth_link *l2; 622 623 l1 = &dev->data->dev_link; 624 l2 = Ð(TX_SUBDEV(dev))->data->dev_link; 625 if (memcmp(l1, l2, sizeof(*l1))) { 626 *l1 = *l2; 627 fs_unlock(dev, 0); 628 return 0; 629 } 630 } 631 fs_unlock(dev, 0); 632 return -1; 633 } 634 635 static int 636 fs_stats_get(struct rte_eth_dev *dev, 637 struct rte_eth_stats *stats) 638 { 639 struct rte_eth_stats backup; 640 struct sub_device *sdev; 641 uint8_t i; 642 int ret; 643 644 fs_lock(dev, 0); 645 rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats)); 646 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 647 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats; 648 uint64_t *timestamp = &sdev->stats_snapshot.timestamp; 649 650 rte_memcpy(&backup, snapshot, sizeof(backup)); 651 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot); 652 if (ret) { 653 if (!fs_err(sdev, ret)) { 654 rte_memcpy(snapshot, &backup, sizeof(backup)); 655 goto inc; 656 } 657 ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d", 658 i, ret); 659 *timestamp = 0; 660 fs_unlock(dev, 0); 661 return ret; 662 } 663 *timestamp = rte_rdtsc(); 664 inc: 665 failsafe_stats_increment(stats, snapshot); 666 } 667 fs_unlock(dev, 0); 668 return 0; 669 } 670 671 static void 672 fs_stats_reset(struct rte_eth_dev *dev) 673 { 674 struct sub_device *sdev; 675 uint8_t i; 676 677 fs_lock(dev, 0); 678 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 679 rte_eth_stats_reset(PORT_ID(sdev)); 680 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats)); 681 } 682 memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats)); 683 fs_unlock(dev, 0); 684 } 685 686 /** 687 * Fail-safe dev_infos_get rules: 688 * 689 * No sub_device: 690 * Numerables: 691 * Use the maximum possible values for any field, so as not 692 * to impede any further configuration effort. 693 * Capabilities: 694 * Limits capabilities to those that are understood by the 695 * fail-safe PMD. This understanding stems from the fail-safe 696 * being capable of verifying that the related capability is 697 * expressed within the device configuration (struct rte_eth_conf). 698 * 699 * At least one probed sub_device: 700 * Numerables: 701 * Uses values from the active probed sub_device 702 * The rationale here is that if any sub_device is less capable 703 * (for example concerning the number of queues) than the active 704 * sub_device, then its subsequent configuration will fail. 705 * It is impossible to foresee this failure when the failing sub_device 706 * is supposed to be plugged-in later on, so the configuration process 707 * is the single point of failure and error reporting. 708 * Capabilities: 709 * Uses a logical AND of RX capabilities among 710 * all sub_devices and the default capabilities. 711 * Uses a logical AND of TX capabilities among 712 * the active probed sub_device and the default capabilities. 713 * 714 */ 715 static void 716 fs_dev_infos_get(struct rte_eth_dev *dev, 717 struct rte_eth_dev_info *infos) 718 { 719 struct sub_device *sdev; 720 uint8_t i; 721 722 sdev = TX_SUBDEV(dev); 723 if (sdev == NULL) { 724 DEBUG("No probed device, using default infos"); 725 rte_memcpy(&PRIV(dev)->infos, &default_infos, 726 sizeof(default_infos)); 727 } else { 728 uint64_t rx_offload_capa; 729 uint64_t rxq_offload_capa; 730 uint64_t rss_hf_offload_capa; 731 732 rx_offload_capa = default_infos.rx_offload_capa; 733 rxq_offload_capa = default_infos.rx_queue_offload_capa; 734 rss_hf_offload_capa = default_infos.flow_type_rss_offloads; 735 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { 736 rte_eth_dev_info_get(PORT_ID(sdev), 737 &PRIV(dev)->infos); 738 rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa; 739 rxq_offload_capa &= 740 PRIV(dev)->infos.rx_queue_offload_capa; 741 rss_hf_offload_capa &= 742 PRIV(dev)->infos.flow_type_rss_offloads; 743 } 744 sdev = TX_SUBDEV(dev); 745 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos); 746 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa; 747 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa; 748 PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa; 749 PRIV(dev)->infos.tx_offload_capa &= 750 default_infos.tx_offload_capa; 751 PRIV(dev)->infos.tx_queue_offload_capa &= 752 default_infos.tx_queue_offload_capa; 753 } 754 rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos)); 755 } 756 757 static const uint32_t * 758 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev) 759 { 760 struct sub_device *sdev; 761 struct rte_eth_dev *edev; 762 const uint32_t *ret; 763 764 fs_lock(dev, 0); 765 sdev = TX_SUBDEV(dev); 766 if (sdev == NULL) { 767 ret = NULL; 768 goto unlock; 769 } 770 edev = ETH(sdev); 771 /* ENOTSUP: counts as no supported ptypes */ 772 if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) { 773 ret = NULL; 774 goto unlock; 775 } 776 /* 777 * The API does not permit to do a clean AND of all ptypes, 778 * It is also incomplete by design and we do not really care 779 * to have a best possible value in this context. 780 * We just return the ptypes of the device of highest 781 * priority, usually the PREFERRED device. 782 */ 783 ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev); 784 unlock: 785 fs_unlock(dev, 0); 786 return ret; 787 } 788 789 static int 790 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 791 { 792 struct sub_device *sdev; 793 uint8_t i; 794 int ret; 795 796 fs_lock(dev, 0); 797 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 798 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i); 799 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu); 800 if ((ret = fs_err(sdev, ret))) { 801 ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d", 802 i, ret); 803 fs_unlock(dev, 0); 804 return ret; 805 } 806 } 807 fs_unlock(dev, 0); 808 return 0; 809 } 810 811 static int 812 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 813 { 814 struct sub_device *sdev; 815 uint8_t i; 816 int ret; 817 818 fs_lock(dev, 0); 819 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 820 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i); 821 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on); 822 if ((ret = fs_err(sdev, ret))) { 823 ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d" 824 " with error %d", i, ret); 825 fs_unlock(dev, 0); 826 return ret; 827 } 828 } 829 fs_unlock(dev, 0); 830 return 0; 831 } 832 833 static int 834 fs_flow_ctrl_get(struct rte_eth_dev *dev, 835 struct rte_eth_fc_conf *fc_conf) 836 { 837 struct sub_device *sdev; 838 int ret; 839 840 fs_lock(dev, 0); 841 sdev = TX_SUBDEV(dev); 842 if (sdev == NULL) { 843 ret = 0; 844 goto unlock; 845 } 846 if (SUBOPS(sdev, flow_ctrl_get) == NULL) { 847 ret = -ENOTSUP; 848 goto unlock; 849 } 850 ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf); 851 unlock: 852 fs_unlock(dev, 0); 853 return ret; 854 } 855 856 static int 857 fs_flow_ctrl_set(struct rte_eth_dev *dev, 858 struct rte_eth_fc_conf *fc_conf) 859 { 860 struct sub_device *sdev; 861 uint8_t i; 862 int ret; 863 864 fs_lock(dev, 0); 865 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 866 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i); 867 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf); 868 if ((ret = fs_err(sdev, ret))) { 869 ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d" 870 " with error %d", i, ret); 871 fs_unlock(dev, 0); 872 return ret; 873 } 874 } 875 fs_unlock(dev, 0); 876 return 0; 877 } 878 879 static void 880 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 881 { 882 struct sub_device *sdev; 883 uint8_t i; 884 885 fs_lock(dev, 0); 886 /* No check: already done within the rte_eth_dev_mac_addr_remove 887 * call for the fail-safe device. 888 */ 889 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 890 rte_eth_dev_mac_addr_remove(PORT_ID(sdev), 891 &dev->data->mac_addrs[index]); 892 PRIV(dev)->mac_addr_pool[index] = 0; 893 fs_unlock(dev, 0); 894 } 895 896 static int 897 fs_mac_addr_add(struct rte_eth_dev *dev, 898 struct ether_addr *mac_addr, 899 uint32_t index, 900 uint32_t vmdq) 901 { 902 struct sub_device *sdev; 903 int ret; 904 uint8_t i; 905 906 RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR); 907 fs_lock(dev, 0); 908 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 909 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq); 910 if ((ret = fs_err(sdev, ret))) { 911 ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %" 912 PRIu8 " with error %d", i, ret); 913 fs_unlock(dev, 0); 914 return ret; 915 } 916 } 917 if (index >= PRIV(dev)->nb_mac_addr) { 918 DEBUG("Growing mac_addrs array"); 919 PRIV(dev)->nb_mac_addr = index; 920 } 921 PRIV(dev)->mac_addr_pool[index] = vmdq; 922 fs_unlock(dev, 0); 923 return 0; 924 } 925 926 static int 927 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 928 { 929 struct sub_device *sdev; 930 uint8_t i; 931 int ret; 932 933 fs_lock(dev, 0); 934 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 935 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr); 936 ret = fs_err(sdev, ret); 937 if (ret) { 938 ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d", 939 i, ret); 940 fs_unlock(dev, 0); 941 return ret; 942 } 943 } 944 fs_unlock(dev, 0); 945 946 return 0; 947 } 948 949 static int 950 fs_filter_ctrl(struct rte_eth_dev *dev, 951 enum rte_filter_type type, 952 enum rte_filter_op op, 953 void *arg) 954 { 955 struct sub_device *sdev; 956 uint8_t i; 957 int ret; 958 959 if (type == RTE_ETH_FILTER_GENERIC && 960 op == RTE_ETH_FILTER_GET) { 961 *(const void **)arg = &fs_flow_ops; 962 return 0; 963 } 964 fs_lock(dev, 0); 965 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 966 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i); 967 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg); 968 if ((ret = fs_err(sdev, ret))) { 969 ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d" 970 " with error %d", i, ret); 971 fs_unlock(dev, 0); 972 return ret; 973 } 974 } 975 fs_unlock(dev, 0); 976 return 0; 977 } 978 979 const struct eth_dev_ops failsafe_ops = { 980 .dev_configure = fs_dev_configure, 981 .dev_start = fs_dev_start, 982 .dev_stop = fs_dev_stop, 983 .dev_set_link_down = fs_dev_set_link_down, 984 .dev_set_link_up = fs_dev_set_link_up, 985 .dev_close = fs_dev_close, 986 .promiscuous_enable = fs_promiscuous_enable, 987 .promiscuous_disable = fs_promiscuous_disable, 988 .allmulticast_enable = fs_allmulticast_enable, 989 .allmulticast_disable = fs_allmulticast_disable, 990 .link_update = fs_link_update, 991 .stats_get = fs_stats_get, 992 .stats_reset = fs_stats_reset, 993 .dev_infos_get = fs_dev_infos_get, 994 .dev_supported_ptypes_get = fs_dev_supported_ptypes_get, 995 .mtu_set = fs_mtu_set, 996 .vlan_filter_set = fs_vlan_filter_set, 997 .rx_queue_setup = fs_rx_queue_setup, 998 .tx_queue_setup = fs_tx_queue_setup, 999 .rx_queue_release = fs_rx_queue_release, 1000 .tx_queue_release = fs_tx_queue_release, 1001 .rx_queue_intr_enable = fs_rx_intr_enable, 1002 .rx_queue_intr_disable = fs_rx_intr_disable, 1003 .flow_ctrl_get = fs_flow_ctrl_get, 1004 .flow_ctrl_set = fs_flow_ctrl_set, 1005 .mac_addr_remove = fs_mac_addr_remove, 1006 .mac_addr_add = fs_mac_addr_add, 1007 .mac_addr_set = fs_mac_addr_set, 1008 .filter_ctrl = fs_filter_ctrl, 1009 }; 1010