1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdbool.h> 35 #include <stdint.h> 36 #include <unistd.h> 37 38 #include <rte_debug.h> 39 #include <rte_atomic.h> 40 #include <rte_ethdev_driver.h> 41 #include <rte_malloc.h> 42 #include <rte_flow.h> 43 #include <rte_cycles.h> 44 45 #include "failsafe_private.h" 46 47 static struct rte_eth_dev_info default_infos = { 48 /* Max possible number of elements */ 49 .max_rx_pktlen = UINT32_MAX, 50 .max_rx_queues = RTE_MAX_QUEUES_PER_PORT, 51 .max_tx_queues = RTE_MAX_QUEUES_PER_PORT, 52 .max_mac_addrs = FAILSAFE_MAX_ETHADDR, 53 .max_hash_mac_addrs = UINT32_MAX, 54 .max_vfs = UINT16_MAX, 55 .max_vmdq_pools = UINT16_MAX, 56 .rx_desc_lim = { 57 .nb_max = UINT16_MAX, 58 .nb_min = 0, 59 .nb_align = 1, 60 .nb_seg_max = UINT16_MAX, 61 .nb_mtu_seg_max = UINT16_MAX, 62 }, 63 .tx_desc_lim = { 64 .nb_max = UINT16_MAX, 65 .nb_min = 0, 66 .nb_align = 1, 67 .nb_seg_max = UINT16_MAX, 68 .nb_mtu_seg_max = UINT16_MAX, 69 }, 70 /* 71 * Set of capabilities that can be verified upon 72 * configuring a sub-device. 73 */ 74 .rx_offload_capa = 75 DEV_RX_OFFLOAD_VLAN_STRIP | 76 DEV_RX_OFFLOAD_IPV4_CKSUM | 77 DEV_RX_OFFLOAD_UDP_CKSUM | 78 DEV_RX_OFFLOAD_TCP_CKSUM | 79 DEV_RX_OFFLOAD_TCP_LRO | 80 DEV_RX_OFFLOAD_QINQ_STRIP | 81 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 82 DEV_RX_OFFLOAD_MACSEC_STRIP | 83 DEV_RX_OFFLOAD_HEADER_SPLIT | 84 DEV_RX_OFFLOAD_VLAN_FILTER | 85 DEV_RX_OFFLOAD_VLAN_EXTEND | 86 DEV_RX_OFFLOAD_JUMBO_FRAME | 87 DEV_RX_OFFLOAD_CRC_STRIP | 88 DEV_RX_OFFLOAD_SCATTER | 89 DEV_RX_OFFLOAD_TIMESTAMP | 90 DEV_RX_OFFLOAD_SECURITY, 91 .rx_queue_offload_capa = 92 DEV_RX_OFFLOAD_VLAN_STRIP | 93 DEV_RX_OFFLOAD_IPV4_CKSUM | 94 DEV_RX_OFFLOAD_UDP_CKSUM | 95 DEV_RX_OFFLOAD_TCP_CKSUM | 96 DEV_RX_OFFLOAD_TCP_LRO | 97 DEV_RX_OFFLOAD_QINQ_STRIP | 98 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 99 DEV_RX_OFFLOAD_MACSEC_STRIP | 100 DEV_RX_OFFLOAD_HEADER_SPLIT | 101 DEV_RX_OFFLOAD_VLAN_FILTER | 102 DEV_RX_OFFLOAD_VLAN_EXTEND | 103 DEV_RX_OFFLOAD_JUMBO_FRAME | 104 DEV_RX_OFFLOAD_CRC_STRIP | 105 DEV_RX_OFFLOAD_SCATTER | 106 DEV_RX_OFFLOAD_TIMESTAMP | 107 DEV_RX_OFFLOAD_SECURITY, 108 .tx_offload_capa = 0x0, 109 .flow_type_rss_offloads = 0x0, 110 }; 111 112 static int 113 fs_dev_configure(struct rte_eth_dev *dev) 114 { 115 struct sub_device *sdev; 116 uint64_t supp_tx_offloads; 117 uint64_t tx_offloads; 118 uint8_t i; 119 int ret; 120 121 supp_tx_offloads = PRIV(dev)->infos.tx_offload_capa; 122 tx_offloads = dev->data->dev_conf.txmode.offloads; 123 if ((tx_offloads & supp_tx_offloads) != tx_offloads) { 124 rte_errno = ENOTSUP; 125 ERROR("Some Tx offloads are not supported, " 126 "requested 0x%" PRIx64 " supported 0x%" PRIx64, 127 tx_offloads, supp_tx_offloads); 128 return -rte_errno; 129 } 130 FOREACH_SUBDEV(sdev, i, dev) { 131 int rmv_interrupt = 0; 132 int lsc_interrupt = 0; 133 int lsc_enabled; 134 135 if (sdev->state != DEV_PROBED) 136 continue; 137 138 rmv_interrupt = ETH(sdev)->data->dev_flags & 139 RTE_ETH_DEV_INTR_RMV; 140 if (rmv_interrupt) { 141 DEBUG("Enabling RMV interrupts for sub_device %d", i); 142 dev->data->dev_conf.intr_conf.rmv = 1; 143 } else { 144 DEBUG("sub_device %d does not support RMV event", i); 145 } 146 lsc_enabled = dev->data->dev_conf.intr_conf.lsc; 147 lsc_interrupt = lsc_enabled && 148 (ETH(sdev)->data->dev_flags & 149 RTE_ETH_DEV_INTR_LSC); 150 if (lsc_interrupt) { 151 DEBUG("Enabling LSC interrupts for sub_device %d", i); 152 dev->data->dev_conf.intr_conf.lsc = 1; 153 } else if (lsc_enabled && !lsc_interrupt) { 154 DEBUG("Disabling LSC interrupts for sub_device %d", i); 155 dev->data->dev_conf.intr_conf.lsc = 0; 156 } 157 DEBUG("Configuring sub-device %d", i); 158 sdev->remove = 0; 159 ret = rte_eth_dev_configure(PORT_ID(sdev), 160 dev->data->nb_rx_queues, 161 dev->data->nb_tx_queues, 162 &dev->data->dev_conf); 163 if (ret) { 164 if (!fs_err(sdev, ret)) 165 continue; 166 ERROR("Could not configure sub_device %d", i); 167 return ret; 168 } 169 if (rmv_interrupt) { 170 ret = rte_eth_dev_callback_register(PORT_ID(sdev), 171 RTE_ETH_EVENT_INTR_RMV, 172 failsafe_eth_rmv_event_callback, 173 sdev); 174 if (ret) 175 WARN("Failed to register RMV callback for sub_device %d", 176 SUB_ID(sdev)); 177 } 178 dev->data->dev_conf.intr_conf.rmv = 0; 179 if (lsc_interrupt) { 180 ret = rte_eth_dev_callback_register(PORT_ID(sdev), 181 RTE_ETH_EVENT_INTR_LSC, 182 failsafe_eth_lsc_event_callback, 183 dev); 184 if (ret) 185 WARN("Failed to register LSC callback for sub_device %d", 186 SUB_ID(sdev)); 187 } 188 dev->data->dev_conf.intr_conf.lsc = lsc_enabled; 189 sdev->state = DEV_ACTIVE; 190 } 191 if (PRIV(dev)->state < DEV_ACTIVE) 192 PRIV(dev)->state = DEV_ACTIVE; 193 return 0; 194 } 195 196 static int 197 fs_dev_start(struct rte_eth_dev *dev) 198 { 199 struct sub_device *sdev; 200 uint8_t i; 201 int ret; 202 203 ret = failsafe_rx_intr_install(dev); 204 if (ret) 205 return ret; 206 FOREACH_SUBDEV(sdev, i, dev) { 207 if (sdev->state != DEV_ACTIVE) 208 continue; 209 DEBUG("Starting sub_device %d", i); 210 ret = rte_eth_dev_start(PORT_ID(sdev)); 211 if (ret) { 212 if (!fs_err(sdev, ret)) 213 continue; 214 return ret; 215 } 216 ret = failsafe_rx_intr_install_subdevice(sdev); 217 if (ret) { 218 if (!fs_err(sdev, ret)) 219 continue; 220 rte_eth_dev_stop(PORT_ID(sdev)); 221 return ret; 222 } 223 sdev->state = DEV_STARTED; 224 } 225 if (PRIV(dev)->state < DEV_STARTED) 226 PRIV(dev)->state = DEV_STARTED; 227 fs_switch_dev(dev, NULL); 228 return 0; 229 } 230 231 static void 232 fs_dev_stop(struct rte_eth_dev *dev) 233 { 234 struct sub_device *sdev; 235 uint8_t i; 236 237 PRIV(dev)->state = DEV_STARTED - 1; 238 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) { 239 rte_eth_dev_stop(PORT_ID(sdev)); 240 failsafe_rx_intr_uninstall_subdevice(sdev); 241 sdev->state = DEV_STARTED - 1; 242 } 243 failsafe_rx_intr_uninstall(dev); 244 } 245 246 static int 247 fs_dev_set_link_up(struct rte_eth_dev *dev) 248 { 249 struct sub_device *sdev; 250 uint8_t i; 251 int ret; 252 253 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 254 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i); 255 ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); 256 if ((ret = fs_err(sdev, ret))) { 257 ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d" 258 " with error %d", i, ret); 259 return ret; 260 } 261 } 262 return 0; 263 } 264 265 static int 266 fs_dev_set_link_down(struct rte_eth_dev *dev) 267 { 268 struct sub_device *sdev; 269 uint8_t i; 270 int ret; 271 272 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 273 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i); 274 ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); 275 if ((ret = fs_err(sdev, ret))) { 276 ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d" 277 " with error %d", i, ret); 278 return ret; 279 } 280 } 281 return 0; 282 } 283 284 static void fs_dev_free_queues(struct rte_eth_dev *dev); 285 static void 286 fs_dev_close(struct rte_eth_dev *dev) 287 { 288 struct sub_device *sdev; 289 uint8_t i; 290 291 failsafe_hotplug_alarm_cancel(dev); 292 if (PRIV(dev)->state == DEV_STARTED) 293 dev->dev_ops->dev_stop(dev); 294 PRIV(dev)->state = DEV_ACTIVE - 1; 295 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 296 DEBUG("Closing sub_device %d", i); 297 rte_eth_dev_close(PORT_ID(sdev)); 298 sdev->state = DEV_ACTIVE - 1; 299 } 300 fs_dev_free_queues(dev); 301 } 302 303 static bool 304 fs_rxq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads) 305 { 306 uint64_t port_offloads; 307 uint64_t queue_supp_offloads; 308 uint64_t port_supp_offloads; 309 310 port_offloads = dev->data->dev_conf.rxmode.offloads; 311 queue_supp_offloads = PRIV(dev)->infos.rx_queue_offload_capa; 312 port_supp_offloads = PRIV(dev)->infos.rx_offload_capa; 313 if ((offloads & (queue_supp_offloads | port_supp_offloads)) != 314 offloads) 315 return false; 316 /* Verify we have no conflict with port offloads */ 317 if ((port_offloads ^ offloads) & port_supp_offloads) 318 return false; 319 return true; 320 } 321 322 static void 323 fs_rx_queue_release(void *queue) 324 { 325 struct rte_eth_dev *dev; 326 struct sub_device *sdev; 327 uint8_t i; 328 struct rxq *rxq; 329 330 if (queue == NULL) 331 return; 332 rxq = queue; 333 if (rxq->event_fd > 0) 334 close(rxq->event_fd); 335 dev = rxq->priv->dev; 336 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 337 SUBOPS(sdev, rx_queue_release) 338 (ETH(sdev)->data->rx_queues[rxq->qid]); 339 dev->data->rx_queues[rxq->qid] = NULL; 340 rte_free(rxq); 341 } 342 343 static int 344 fs_rx_queue_setup(struct rte_eth_dev *dev, 345 uint16_t rx_queue_id, 346 uint16_t nb_rx_desc, 347 unsigned int socket_id, 348 const struct rte_eth_rxconf *rx_conf, 349 struct rte_mempool *mb_pool) 350 { 351 /* 352 * FIXME: Add a proper interface in rte_eal_interrupts for 353 * allocating eventfd as an interrupt vector. 354 * For the time being, fake as if we are using MSIX interrupts, 355 * this will cause rte_intr_efd_enable to allocate an eventfd for us. 356 */ 357 struct rte_intr_handle intr_handle = { 358 .type = RTE_INTR_HANDLE_VFIO_MSIX, 359 .efds = { -1, }, 360 }; 361 struct sub_device *sdev; 362 struct rxq *rxq; 363 uint8_t i; 364 int ret; 365 366 rxq = dev->data->rx_queues[rx_queue_id]; 367 if (rxq != NULL) { 368 fs_rx_queue_release(rxq); 369 dev->data->rx_queues[rx_queue_id] = NULL; 370 } 371 /* Verify application offloads are valid for our port and queue. */ 372 if (fs_rxq_offloads_valid(dev, rx_conf->offloads) == false) { 373 rte_errno = ENOTSUP; 374 ERROR("Rx queue offloads 0x%" PRIx64 375 " don't match port offloads 0x%" PRIx64 376 " or supported offloads 0x%" PRIx64, 377 rx_conf->offloads, 378 dev->data->dev_conf.rxmode.offloads, 379 PRIV(dev)->infos.rx_offload_capa | 380 PRIV(dev)->infos.rx_queue_offload_capa); 381 return -rte_errno; 382 } 383 rxq = rte_zmalloc(NULL, 384 sizeof(*rxq) + 385 sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, 386 RTE_CACHE_LINE_SIZE); 387 if (rxq == NULL) 388 return -ENOMEM; 389 FOREACH_SUBDEV(sdev, i, dev) 390 rte_atomic64_init(&rxq->refcnt[i]); 391 rxq->qid = rx_queue_id; 392 rxq->socket_id = socket_id; 393 rxq->info.mp = mb_pool; 394 rxq->info.conf = *rx_conf; 395 rxq->info.nb_desc = nb_rx_desc; 396 rxq->priv = PRIV(dev); 397 rxq->sdev = PRIV(dev)->subs; 398 ret = rte_intr_efd_enable(&intr_handle, 1); 399 if (ret < 0) 400 return ret; 401 rxq->event_fd = intr_handle.efds[0]; 402 dev->data->rx_queues[rx_queue_id] = rxq; 403 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 404 ret = rte_eth_rx_queue_setup(PORT_ID(sdev), 405 rx_queue_id, 406 nb_rx_desc, socket_id, 407 rx_conf, mb_pool); 408 if ((ret = fs_err(sdev, ret))) { 409 ERROR("RX queue setup failed for sub_device %d", i); 410 goto free_rxq; 411 } 412 } 413 return 0; 414 free_rxq: 415 fs_rx_queue_release(rxq); 416 return ret; 417 } 418 419 static int 420 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) 421 { 422 struct rxq *rxq; 423 struct sub_device *sdev; 424 uint8_t i; 425 int ret; 426 int rc = 0; 427 428 if (idx >= dev->data->nb_rx_queues) { 429 rte_errno = EINVAL; 430 return -rte_errno; 431 } 432 rxq = dev->data->rx_queues[idx]; 433 if (rxq == NULL || rxq->event_fd <= 0) { 434 rte_errno = EINVAL; 435 return -rte_errno; 436 } 437 /* Fail if proxy service is nor running. */ 438 if (PRIV(dev)->rxp.sstate != SS_RUNNING) { 439 ERROR("failsafe interrupt services are not running"); 440 rte_errno = EAGAIN; 441 return -rte_errno; 442 } 443 rxq->enable_events = 1; 444 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 445 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx); 446 ret = fs_err(sdev, ret); 447 if (ret) 448 rc = ret; 449 } 450 if (rc) 451 rte_errno = -rc; 452 return rc; 453 } 454 455 static int 456 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx) 457 { 458 struct rxq *rxq; 459 struct sub_device *sdev; 460 uint64_t u64; 461 uint8_t i; 462 int rc = 0; 463 int ret; 464 465 if (idx >= dev->data->nb_rx_queues) { 466 rte_errno = EINVAL; 467 return -rte_errno; 468 } 469 rxq = dev->data->rx_queues[idx]; 470 if (rxq == NULL || rxq->event_fd <= 0) { 471 rte_errno = EINVAL; 472 return -rte_errno; 473 } 474 rxq->enable_events = 0; 475 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 476 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx); 477 ret = fs_err(sdev, ret); 478 if (ret) 479 rc = ret; 480 } 481 /* Clear pending events */ 482 while (read(rxq->event_fd, &u64, sizeof(uint64_t)) > 0) 483 ; 484 if (rc) 485 rte_errno = -rc; 486 return rc; 487 } 488 489 static bool 490 fs_txq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads) 491 { 492 uint64_t port_offloads; 493 uint64_t queue_supp_offloads; 494 uint64_t port_supp_offloads; 495 496 port_offloads = dev->data->dev_conf.txmode.offloads; 497 queue_supp_offloads = PRIV(dev)->infos.tx_queue_offload_capa; 498 port_supp_offloads = PRIV(dev)->infos.tx_offload_capa; 499 if ((offloads & (queue_supp_offloads | port_supp_offloads)) != 500 offloads) 501 return false; 502 /* Verify we have no conflict with port offloads */ 503 if ((port_offloads ^ offloads) & port_supp_offloads) 504 return false; 505 return true; 506 } 507 508 static void 509 fs_tx_queue_release(void *queue) 510 { 511 struct rte_eth_dev *dev; 512 struct sub_device *sdev; 513 uint8_t i; 514 struct txq *txq; 515 516 if (queue == NULL) 517 return; 518 txq = queue; 519 dev = txq->priv->dev; 520 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 521 SUBOPS(sdev, tx_queue_release) 522 (ETH(sdev)->data->tx_queues[txq->qid]); 523 dev->data->tx_queues[txq->qid] = NULL; 524 rte_free(txq); 525 } 526 527 static int 528 fs_tx_queue_setup(struct rte_eth_dev *dev, 529 uint16_t tx_queue_id, 530 uint16_t nb_tx_desc, 531 unsigned int socket_id, 532 const struct rte_eth_txconf *tx_conf) 533 { 534 struct sub_device *sdev; 535 struct txq *txq; 536 uint8_t i; 537 int ret; 538 539 txq = dev->data->tx_queues[tx_queue_id]; 540 if (txq != NULL) { 541 fs_tx_queue_release(txq); 542 dev->data->tx_queues[tx_queue_id] = NULL; 543 } 544 /* 545 * Don't verify queue offloads for applications which 546 * use the old API. 547 */ 548 if (tx_conf != NULL && 549 (tx_conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) && 550 fs_txq_offloads_valid(dev, tx_conf->offloads) == false) { 551 rte_errno = ENOTSUP; 552 ERROR("Tx queue offloads 0x%" PRIx64 553 " don't match port offloads 0x%" PRIx64 554 " or supported offloads 0x%" PRIx64, 555 tx_conf->offloads, 556 dev->data->dev_conf.txmode.offloads, 557 PRIV(dev)->infos.tx_offload_capa | 558 PRIV(dev)->infos.tx_queue_offload_capa); 559 return -rte_errno; 560 } 561 txq = rte_zmalloc("ethdev TX queue", 562 sizeof(*txq) + 563 sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, 564 RTE_CACHE_LINE_SIZE); 565 if (txq == NULL) 566 return -ENOMEM; 567 FOREACH_SUBDEV(sdev, i, dev) 568 rte_atomic64_init(&txq->refcnt[i]); 569 txq->qid = tx_queue_id; 570 txq->socket_id = socket_id; 571 txq->info.conf = *tx_conf; 572 txq->info.nb_desc = nb_tx_desc; 573 txq->priv = PRIV(dev); 574 dev->data->tx_queues[tx_queue_id] = txq; 575 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 576 ret = rte_eth_tx_queue_setup(PORT_ID(sdev), 577 tx_queue_id, 578 nb_tx_desc, socket_id, 579 tx_conf); 580 if ((ret = fs_err(sdev, ret))) { 581 ERROR("TX queue setup failed for sub_device %d", i); 582 goto free_txq; 583 } 584 } 585 return 0; 586 free_txq: 587 fs_tx_queue_release(txq); 588 return ret; 589 } 590 591 static void 592 fs_dev_free_queues(struct rte_eth_dev *dev) 593 { 594 uint16_t i; 595 596 for (i = 0; i < dev->data->nb_rx_queues; i++) { 597 fs_rx_queue_release(dev->data->rx_queues[i]); 598 dev->data->rx_queues[i] = NULL; 599 } 600 dev->data->nb_rx_queues = 0; 601 for (i = 0; i < dev->data->nb_tx_queues; i++) { 602 fs_tx_queue_release(dev->data->tx_queues[i]); 603 dev->data->tx_queues[i] = NULL; 604 } 605 dev->data->nb_tx_queues = 0; 606 } 607 608 static void 609 fs_promiscuous_enable(struct rte_eth_dev *dev) 610 { 611 struct sub_device *sdev; 612 uint8_t i; 613 614 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 615 rte_eth_promiscuous_enable(PORT_ID(sdev)); 616 } 617 618 static void 619 fs_promiscuous_disable(struct rte_eth_dev *dev) 620 { 621 struct sub_device *sdev; 622 uint8_t i; 623 624 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 625 rte_eth_promiscuous_disable(PORT_ID(sdev)); 626 } 627 628 static void 629 fs_allmulticast_enable(struct rte_eth_dev *dev) 630 { 631 struct sub_device *sdev; 632 uint8_t i; 633 634 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 635 rte_eth_allmulticast_enable(PORT_ID(sdev)); 636 } 637 638 static void 639 fs_allmulticast_disable(struct rte_eth_dev *dev) 640 { 641 struct sub_device *sdev; 642 uint8_t i; 643 644 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 645 rte_eth_allmulticast_disable(PORT_ID(sdev)); 646 } 647 648 static int 649 fs_link_update(struct rte_eth_dev *dev, 650 int wait_to_complete) 651 { 652 struct sub_device *sdev; 653 uint8_t i; 654 int ret; 655 656 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 657 DEBUG("Calling link_update on sub_device %d", i); 658 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete); 659 if (ret && ret != -1 && sdev->remove == 0 && 660 rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) { 661 ERROR("Link update failed for sub_device %d with error %d", 662 i, ret); 663 return ret; 664 } 665 } 666 if (TX_SUBDEV(dev)) { 667 struct rte_eth_link *l1; 668 struct rte_eth_link *l2; 669 670 l1 = &dev->data->dev_link; 671 l2 = Ð(TX_SUBDEV(dev))->data->dev_link; 672 if (memcmp(l1, l2, sizeof(*l1))) { 673 *l1 = *l2; 674 return 0; 675 } 676 } 677 return -1; 678 } 679 680 static int 681 fs_stats_get(struct rte_eth_dev *dev, 682 struct rte_eth_stats *stats) 683 { 684 struct rte_eth_stats backup; 685 struct sub_device *sdev; 686 uint8_t i; 687 int ret; 688 689 rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats)); 690 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 691 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats; 692 uint64_t *timestamp = &sdev->stats_snapshot.timestamp; 693 694 rte_memcpy(&backup, snapshot, sizeof(backup)); 695 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot); 696 if (ret) { 697 if (!fs_err(sdev, ret)) { 698 rte_memcpy(snapshot, &backup, sizeof(backup)); 699 goto inc; 700 } 701 ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d", 702 i, ret); 703 *timestamp = 0; 704 return ret; 705 } 706 *timestamp = rte_rdtsc(); 707 inc: 708 failsafe_stats_increment(stats, snapshot); 709 } 710 return 0; 711 } 712 713 static void 714 fs_stats_reset(struct rte_eth_dev *dev) 715 { 716 struct sub_device *sdev; 717 uint8_t i; 718 719 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 720 rte_eth_stats_reset(PORT_ID(sdev)); 721 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats)); 722 } 723 memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats)); 724 } 725 726 /** 727 * Fail-safe dev_infos_get rules: 728 * 729 * No sub_device: 730 * Numerables: 731 * Use the maximum possible values for any field, so as not 732 * to impede any further configuration effort. 733 * Capabilities: 734 * Limits capabilities to those that are understood by the 735 * fail-safe PMD. This understanding stems from the fail-safe 736 * being capable of verifying that the related capability is 737 * expressed within the device configuration (struct rte_eth_conf). 738 * 739 * At least one probed sub_device: 740 * Numerables: 741 * Uses values from the active probed sub_device 742 * The rationale here is that if any sub_device is less capable 743 * (for example concerning the number of queues) than the active 744 * sub_device, then its subsequent configuration will fail. 745 * It is impossible to foresee this failure when the failing sub_device 746 * is supposed to be plugged-in later on, so the configuration process 747 * is the single point of failure and error reporting. 748 * Capabilities: 749 * Uses a logical AND of RX capabilities among 750 * all sub_devices and the default capabilities. 751 * Uses a logical AND of TX capabilities among 752 * the active probed sub_device and the default capabilities. 753 * 754 */ 755 static void 756 fs_dev_infos_get(struct rte_eth_dev *dev, 757 struct rte_eth_dev_info *infos) 758 { 759 struct sub_device *sdev; 760 uint8_t i; 761 762 sdev = TX_SUBDEV(dev); 763 if (sdev == NULL) { 764 DEBUG("No probed device, using default infos"); 765 rte_memcpy(&PRIV(dev)->infos, &default_infos, 766 sizeof(default_infos)); 767 } else { 768 uint64_t rx_offload_capa; 769 uint64_t rxq_offload_capa; 770 771 rx_offload_capa = default_infos.rx_offload_capa; 772 rxq_offload_capa = default_infos.rx_queue_offload_capa; 773 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { 774 rte_eth_dev_info_get(PORT_ID(sdev), 775 &PRIV(dev)->infos); 776 rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa; 777 rxq_offload_capa &= 778 PRIV(dev)->infos.rx_queue_offload_capa; 779 } 780 sdev = TX_SUBDEV(dev); 781 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos); 782 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa; 783 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa; 784 PRIV(dev)->infos.tx_offload_capa &= 785 default_infos.tx_offload_capa; 786 PRIV(dev)->infos.tx_queue_offload_capa &= 787 default_infos.tx_queue_offload_capa; 788 PRIV(dev)->infos.flow_type_rss_offloads &= 789 default_infos.flow_type_rss_offloads; 790 } 791 rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos)); 792 } 793 794 static const uint32_t * 795 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev) 796 { 797 struct sub_device *sdev; 798 struct rte_eth_dev *edev; 799 800 sdev = TX_SUBDEV(dev); 801 if (sdev == NULL) 802 return NULL; 803 edev = ETH(sdev); 804 /* ENOTSUP: counts as no supported ptypes */ 805 if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) 806 return NULL; 807 /* 808 * The API does not permit to do a clean AND of all ptypes, 809 * It is also incomplete by design and we do not really care 810 * to have a best possible value in this context. 811 * We just return the ptypes of the device of highest 812 * priority, usually the PREFERRED device. 813 */ 814 return SUBOPS(sdev, dev_supported_ptypes_get)(edev); 815 } 816 817 static int 818 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 819 { 820 struct sub_device *sdev; 821 uint8_t i; 822 int ret; 823 824 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 825 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i); 826 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu); 827 if ((ret = fs_err(sdev, ret))) { 828 ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d", 829 i, ret); 830 return ret; 831 } 832 } 833 return 0; 834 } 835 836 static int 837 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 838 { 839 struct sub_device *sdev; 840 uint8_t i; 841 int ret; 842 843 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 844 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i); 845 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on); 846 if ((ret = fs_err(sdev, ret))) { 847 ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d" 848 " with error %d", i, ret); 849 return ret; 850 } 851 } 852 return 0; 853 } 854 855 static int 856 fs_flow_ctrl_get(struct rte_eth_dev *dev, 857 struct rte_eth_fc_conf *fc_conf) 858 { 859 struct sub_device *sdev; 860 861 sdev = TX_SUBDEV(dev); 862 if (sdev == NULL) 863 return 0; 864 if (SUBOPS(sdev, flow_ctrl_get) == NULL) 865 return -ENOTSUP; 866 return SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf); 867 } 868 869 static int 870 fs_flow_ctrl_set(struct rte_eth_dev *dev, 871 struct rte_eth_fc_conf *fc_conf) 872 { 873 struct sub_device *sdev; 874 uint8_t i; 875 int ret; 876 877 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 878 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i); 879 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf); 880 if ((ret = fs_err(sdev, ret))) { 881 ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d" 882 " with error %d", i, ret); 883 return ret; 884 } 885 } 886 return 0; 887 } 888 889 static void 890 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 891 { 892 struct sub_device *sdev; 893 uint8_t i; 894 895 /* No check: already done within the rte_eth_dev_mac_addr_remove 896 * call for the fail-safe device. 897 */ 898 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 899 rte_eth_dev_mac_addr_remove(PORT_ID(sdev), 900 &dev->data->mac_addrs[index]); 901 PRIV(dev)->mac_addr_pool[index] = 0; 902 } 903 904 static int 905 fs_mac_addr_add(struct rte_eth_dev *dev, 906 struct ether_addr *mac_addr, 907 uint32_t index, 908 uint32_t vmdq) 909 { 910 struct sub_device *sdev; 911 int ret; 912 uint8_t i; 913 914 RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR); 915 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 916 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq); 917 if ((ret = fs_err(sdev, ret))) { 918 ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %" 919 PRIu8 " with error %d", i, ret); 920 return ret; 921 } 922 } 923 if (index >= PRIV(dev)->nb_mac_addr) { 924 DEBUG("Growing mac_addrs array"); 925 PRIV(dev)->nb_mac_addr = index; 926 } 927 PRIV(dev)->mac_addr_pool[index] = vmdq; 928 return 0; 929 } 930 931 static void 932 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 933 { 934 struct sub_device *sdev; 935 uint8_t i; 936 937 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) 938 rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr); 939 } 940 941 static int 942 fs_filter_ctrl(struct rte_eth_dev *dev, 943 enum rte_filter_type type, 944 enum rte_filter_op op, 945 void *arg) 946 { 947 struct sub_device *sdev; 948 uint8_t i; 949 int ret; 950 951 if (type == RTE_ETH_FILTER_GENERIC && 952 op == RTE_ETH_FILTER_GET) { 953 *(const void **)arg = &fs_flow_ops; 954 return 0; 955 } 956 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { 957 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i); 958 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg); 959 if ((ret = fs_err(sdev, ret))) { 960 ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d" 961 " with error %d", i, ret); 962 return ret; 963 } 964 } 965 return 0; 966 } 967 968 const struct eth_dev_ops failsafe_ops = { 969 .dev_configure = fs_dev_configure, 970 .dev_start = fs_dev_start, 971 .dev_stop = fs_dev_stop, 972 .dev_set_link_down = fs_dev_set_link_down, 973 .dev_set_link_up = fs_dev_set_link_up, 974 .dev_close = fs_dev_close, 975 .promiscuous_enable = fs_promiscuous_enable, 976 .promiscuous_disable = fs_promiscuous_disable, 977 .allmulticast_enable = fs_allmulticast_enable, 978 .allmulticast_disable = fs_allmulticast_disable, 979 .link_update = fs_link_update, 980 .stats_get = fs_stats_get, 981 .stats_reset = fs_stats_reset, 982 .dev_infos_get = fs_dev_infos_get, 983 .dev_supported_ptypes_get = fs_dev_supported_ptypes_get, 984 .mtu_set = fs_mtu_set, 985 .vlan_filter_set = fs_vlan_filter_set, 986 .rx_queue_setup = fs_rx_queue_setup, 987 .tx_queue_setup = fs_tx_queue_setup, 988 .rx_queue_release = fs_rx_queue_release, 989 .tx_queue_release = fs_tx_queue_release, 990 .rx_queue_intr_enable = fs_rx_intr_enable, 991 .rx_queue_intr_disable = fs_rx_intr_disable, 992 .flow_ctrl_get = fs_flow_ctrl_get, 993 .flow_ctrl_set = fs_flow_ctrl_set, 994 .mac_addr_remove = fs_mac_addr_remove, 995 .mac_addr_add = fs_mac_addr_add, 996 .mac_addr_set = fs_mac_addr_set, 997 .filter_ctrl = fs_filter_ctrl, 998 }; 999