1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2022 Intel Corporation 3 */ 4 5 #include "gve_ethdev.h" 6 #include "base/gve_adminq.h" 7 #include "base/gve_register.h" 8 #include "base/gve_osdep.h" 9 #include "gve_version.h" 10 11 static void 12 gve_write_version(uint8_t *driver_version_register) 13 { 14 const char *c = gve_version_string(); 15 while (*c) { 16 writeb(*c, driver_version_register); 17 c++; 18 } 19 writeb('\n', driver_version_register); 20 } 21 22 static int 23 gve_alloc_queue_page_list(struct gve_priv *priv, uint32_t id, uint32_t pages) 24 { 25 char z_name[RTE_MEMZONE_NAMESIZE]; 26 struct gve_queue_page_list *qpl; 27 const struct rte_memzone *mz; 28 dma_addr_t page_bus; 29 uint32_t i; 30 31 if (priv->num_registered_pages + pages > 32 priv->max_registered_pages) { 33 PMD_DRV_LOG(ERR, "Pages %" PRIu64 " > max registered pages %" PRIu64, 34 priv->num_registered_pages + pages, 35 priv->max_registered_pages); 36 return -EINVAL; 37 } 38 qpl = &priv->qpl[id]; 39 snprintf(z_name, sizeof(z_name), "gve_%s_qpl%d", priv->pci_dev->device.name, id); 40 mz = rte_memzone_reserve_aligned(z_name, pages * PAGE_SIZE, 41 rte_socket_id(), 42 RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); 43 if (mz == NULL) { 44 PMD_DRV_LOG(ERR, "Failed to alloc %s.", z_name); 45 return -ENOMEM; 46 } 47 qpl->page_buses = rte_zmalloc("qpl page buses", pages * sizeof(dma_addr_t), 0); 48 if (qpl->page_buses == NULL) { 49 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u page buses", id); 50 return -ENOMEM; 51 } 52 page_bus = mz->iova; 53 for (i = 0; i < pages; i++) { 54 qpl->page_buses[i] = page_bus; 55 page_bus += PAGE_SIZE; 56 } 57 qpl->id = id; 58 qpl->mz = mz; 59 qpl->num_entries = pages; 60 61 priv->num_registered_pages += pages; 62 63 return 0; 64 } 65 66 static void 67 gve_free_qpls(struct gve_priv *priv) 68 { 69 uint16_t nb_txqs = priv->max_nb_txq; 70 uint16_t nb_rxqs = priv->max_nb_rxq; 71 uint32_t i; 72 73 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 74 return; 75 76 for (i = 0; i < nb_txqs + nb_rxqs; i++) { 77 if (priv->qpl[i].mz != NULL) 78 rte_memzone_free(priv->qpl[i].mz); 79 rte_free(priv->qpl[i].page_buses); 80 } 81 82 rte_free(priv->qpl); 83 } 84 85 static int 86 gve_dev_configure(struct rte_eth_dev *dev) 87 { 88 struct gve_priv *priv = dev->data->dev_private; 89 90 if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) 91 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; 92 93 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) 94 priv->enable_rsc = 1; 95 96 return 0; 97 } 98 99 static int 100 gve_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 101 { 102 struct gve_priv *priv = dev->data->dev_private; 103 struct rte_eth_link link; 104 int err; 105 106 memset(&link, 0, sizeof(link)); 107 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 108 link.link_autoneg = RTE_ETH_LINK_AUTONEG; 109 110 if (!dev->data->dev_started) { 111 link.link_status = RTE_ETH_LINK_DOWN; 112 link.link_speed = RTE_ETH_SPEED_NUM_NONE; 113 } else { 114 link.link_status = RTE_ETH_LINK_UP; 115 PMD_DRV_LOG(DEBUG, "Get link status from hw"); 116 err = gve_adminq_report_link_speed(priv); 117 if (err) { 118 PMD_DRV_LOG(ERR, "Failed to get link speed."); 119 priv->link_speed = RTE_ETH_SPEED_NUM_UNKNOWN; 120 } 121 link.link_speed = priv->link_speed; 122 } 123 124 return rte_eth_linkstatus_set(dev, &link); 125 } 126 127 static int 128 gve_start_queues(struct rte_eth_dev *dev) 129 { 130 struct gve_priv *priv = dev->data->dev_private; 131 uint16_t num_queues; 132 uint16_t i; 133 int ret; 134 135 num_queues = dev->data->nb_tx_queues; 136 priv->txqs = (struct gve_tx_queue **)dev->data->tx_queues; 137 ret = gve_adminq_create_tx_queues(priv, num_queues); 138 if (ret != 0) { 139 PMD_DRV_LOG(ERR, "Failed to create %u tx queues.", num_queues); 140 return ret; 141 } 142 for (i = 0; i < num_queues; i++) 143 if (gve_tx_queue_start(dev, i) != 0) { 144 PMD_DRV_LOG(ERR, "Fail to start Tx queue %d", i); 145 goto err_tx; 146 } 147 148 num_queues = dev->data->nb_rx_queues; 149 priv->rxqs = (struct gve_rx_queue **)dev->data->rx_queues; 150 ret = gve_adminq_create_rx_queues(priv, num_queues); 151 if (ret != 0) { 152 PMD_DRV_LOG(ERR, "Failed to create %u rx queues.", num_queues); 153 goto err_tx; 154 } 155 for (i = 0; i < num_queues; i++) { 156 if (gve_is_gqi(priv)) 157 ret = gve_rx_queue_start(dev, i); 158 else 159 ret = gve_rx_queue_start_dqo(dev, i); 160 if (ret != 0) { 161 PMD_DRV_LOG(ERR, "Fail to start Rx queue %d", i); 162 goto err_rx; 163 } 164 } 165 166 return 0; 167 168 err_rx: 169 gve_stop_rx_queues(dev); 170 err_tx: 171 gve_stop_tx_queues(dev); 172 return ret; 173 } 174 175 static int 176 gve_dev_start(struct rte_eth_dev *dev) 177 { 178 int ret; 179 180 ret = gve_start_queues(dev); 181 if (ret != 0) { 182 PMD_DRV_LOG(ERR, "Failed to start queues"); 183 return ret; 184 } 185 186 dev->data->dev_started = 1; 187 gve_link_update(dev, 0); 188 189 return 0; 190 } 191 192 static int 193 gve_dev_stop(struct rte_eth_dev *dev) 194 { 195 dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; 196 197 gve_stop_tx_queues(dev); 198 gve_stop_rx_queues(dev); 199 200 dev->data->dev_started = 0; 201 202 return 0; 203 } 204 205 static int 206 gve_dev_close(struct rte_eth_dev *dev) 207 { 208 struct gve_priv *priv = dev->data->dev_private; 209 int err = 0; 210 uint16_t i; 211 212 if (dev->data->dev_started) { 213 err = gve_dev_stop(dev); 214 if (err != 0) 215 PMD_DRV_LOG(ERR, "Failed to stop dev."); 216 } 217 218 if (gve_is_gqi(priv)) { 219 for (i = 0; i < dev->data->nb_tx_queues; i++) 220 gve_tx_queue_release(dev, i); 221 222 for (i = 0; i < dev->data->nb_rx_queues; i++) 223 gve_rx_queue_release(dev, i); 224 } else { 225 for (i = 0; i < dev->data->nb_tx_queues; i++) 226 gve_tx_queue_release_dqo(dev, i); 227 228 for (i = 0; i < dev->data->nb_rx_queues; i++) 229 gve_rx_queue_release_dqo(dev, i); 230 } 231 232 gve_free_qpls(priv); 233 rte_free(priv->adminq); 234 235 dev->data->mac_addrs = NULL; 236 237 return err; 238 } 239 240 static int 241 gve_verify_driver_compatibility(struct gve_priv *priv) 242 { 243 const struct rte_memzone *driver_info_mem; 244 struct gve_driver_info *driver_info; 245 int err; 246 247 driver_info_mem = rte_memzone_reserve_aligned("verify_driver_compatibility", 248 sizeof(struct gve_driver_info), 249 rte_socket_id(), 250 RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); 251 252 if (driver_info_mem == NULL) { 253 PMD_DRV_LOG(ERR, 254 "Could not alloc memzone for driver compatibility"); 255 return -ENOMEM; 256 } 257 driver_info = (struct gve_driver_info *)driver_info_mem->addr; 258 259 *driver_info = (struct gve_driver_info) { 260 .os_type = 5, /* DPDK */ 261 .driver_major = GVE_VERSION_MAJOR, 262 .driver_minor = GVE_VERSION_MINOR, 263 .driver_sub = GVE_VERSION_SUB, 264 .os_version_major = cpu_to_be32(DPDK_VERSION_MAJOR), 265 .os_version_minor = cpu_to_be32(DPDK_VERSION_MINOR), 266 .os_version_sub = cpu_to_be32(DPDK_VERSION_SUB), 267 .driver_capability_flags = { 268 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1), 269 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2), 270 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3), 271 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4), 272 }, 273 }; 274 275 populate_driver_version_strings((char *)driver_info->os_version_str1, 276 (char *)driver_info->os_version_str2); 277 278 err = gve_adminq_verify_driver_compatibility(priv, 279 sizeof(struct gve_driver_info), 280 (dma_addr_t)driver_info_mem->iova); 281 /* It's ok if the device doesn't support this */ 282 if (err == -EOPNOTSUPP) 283 err = 0; 284 285 rte_memzone_free(driver_info_mem); 286 return err; 287 } 288 289 static int 290 gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 291 { 292 struct gve_priv *priv = dev->data->dev_private; 293 294 dev_info->device = dev->device; 295 dev_info->max_mac_addrs = 1; 296 dev_info->max_rx_queues = priv->max_nb_rxq; 297 dev_info->max_tx_queues = priv->max_nb_txq; 298 dev_info->min_rx_bufsize = GVE_MIN_BUF_SIZE; 299 dev_info->max_rx_pktlen = GVE_MAX_RX_PKTLEN; 300 dev_info->max_mtu = GVE_MAX_MTU; 301 dev_info->min_mtu = GVE_MIN_MTU; 302 303 dev_info->rx_offload_capa = 0; 304 dev_info->tx_offload_capa = 305 RTE_ETH_TX_OFFLOAD_MULTI_SEGS | 306 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 307 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 308 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 309 RTE_ETH_TX_OFFLOAD_TCP_TSO; 310 311 if (priv->queue_format == GVE_DQO_RDA_FORMAT) 312 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO; 313 314 dev_info->default_rxconf = (struct rte_eth_rxconf) { 315 .rx_free_thresh = GVE_DEFAULT_RX_FREE_THRESH, 316 .rx_drop_en = 0, 317 .offloads = 0, 318 }; 319 320 dev_info->default_txconf = (struct rte_eth_txconf) { 321 .tx_free_thresh = GVE_DEFAULT_TX_FREE_THRESH, 322 .tx_rs_thresh = GVE_DEFAULT_TX_RS_THRESH, 323 .offloads = 0, 324 }; 325 326 dev_info->default_rxportconf.ring_size = priv->rx_desc_cnt; 327 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 328 .nb_max = priv->rx_desc_cnt, 329 .nb_min = priv->rx_desc_cnt, 330 .nb_align = 1, 331 }; 332 333 dev_info->default_txportconf.ring_size = priv->tx_desc_cnt; 334 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 335 .nb_max = priv->tx_desc_cnt, 336 .nb_min = priv->tx_desc_cnt, 337 .nb_align = 1, 338 }; 339 340 return 0; 341 } 342 343 static int 344 gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 345 { 346 uint16_t i; 347 348 for (i = 0; i < dev->data->nb_tx_queues; i++) { 349 struct gve_tx_queue *txq = dev->data->tx_queues[i]; 350 if (txq == NULL) 351 continue; 352 353 stats->opackets += txq->stats.packets; 354 stats->obytes += txq->stats.bytes; 355 stats->oerrors += txq->stats.errors; 356 } 357 358 for (i = 0; i < dev->data->nb_rx_queues; i++) { 359 struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 360 if (rxq == NULL) 361 continue; 362 363 stats->ipackets += rxq->stats.packets; 364 stats->ibytes += rxq->stats.bytes; 365 stats->ierrors += rxq->stats.errors; 366 stats->rx_nombuf += rxq->stats.no_mbufs; 367 } 368 369 return 0; 370 } 371 372 static int 373 gve_dev_stats_reset(struct rte_eth_dev *dev) 374 { 375 uint16_t i; 376 377 for (i = 0; i < dev->data->nb_tx_queues; i++) { 378 struct gve_tx_queue *txq = dev->data->tx_queues[i]; 379 if (txq == NULL) 380 continue; 381 382 memset(&txq->stats, 0, sizeof(txq->stats)); 383 } 384 385 for (i = 0; i < dev->data->nb_rx_queues; i++) { 386 struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 387 if (rxq == NULL) 388 continue; 389 390 memset(&rxq->stats, 0, sizeof(rxq->stats)); 391 } 392 393 return 0; 394 } 395 396 static int 397 gve_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 398 { 399 struct gve_priv *priv = dev->data->dev_private; 400 int err; 401 402 if (mtu < RTE_ETHER_MIN_MTU || mtu > priv->max_mtu) { 403 PMD_DRV_LOG(ERR, "MIN MTU is %u, MAX MTU is %u", 404 RTE_ETHER_MIN_MTU, priv->max_mtu); 405 return -EINVAL; 406 } 407 408 /* mtu setting is forbidden if port is start */ 409 if (dev->data->dev_started) { 410 PMD_DRV_LOG(ERR, "Port must be stopped before configuration"); 411 return -EBUSY; 412 } 413 414 err = gve_adminq_set_mtu(priv, mtu); 415 if (err) { 416 PMD_DRV_LOG(ERR, "Failed to set mtu as %u err = %d", mtu, err); 417 return err; 418 } 419 420 return 0; 421 } 422 423 #define TX_QUEUE_STATS_OFFSET(x) offsetof(struct gve_tx_stats, x) 424 #define RX_QUEUE_STATS_OFFSET(x) offsetof(struct gve_rx_stats, x) 425 426 static const struct gve_xstats_name_offset tx_xstats_name_offset[] = { 427 { "packets", TX_QUEUE_STATS_OFFSET(packets) }, 428 { "bytes", TX_QUEUE_STATS_OFFSET(bytes) }, 429 { "errors", TX_QUEUE_STATS_OFFSET(errors) }, 430 }; 431 432 static const struct gve_xstats_name_offset rx_xstats_name_offset[] = { 433 { "packets", RX_QUEUE_STATS_OFFSET(packets) }, 434 { "bytes", RX_QUEUE_STATS_OFFSET(bytes) }, 435 { "errors", RX_QUEUE_STATS_OFFSET(errors) }, 436 { "mbuf_alloc_errors", RX_QUEUE_STATS_OFFSET(no_mbufs) }, 437 { "mbuf_alloc_errors_bulk", RX_QUEUE_STATS_OFFSET(no_mbufs_bulk) }, 438 }; 439 440 static int 441 gve_xstats_count(struct rte_eth_dev *dev) 442 { 443 uint16_t i, count = 0; 444 445 for (i = 0; i < dev->data->nb_tx_queues; i++) { 446 if (dev->data->tx_queues[i]) 447 count += RTE_DIM(tx_xstats_name_offset); 448 } 449 450 for (i = 0; i < dev->data->nb_rx_queues; i++) { 451 if (dev->data->rx_queues[i]) 452 count += RTE_DIM(rx_xstats_name_offset); 453 } 454 455 return count; 456 } 457 458 static int 459 gve_xstats_get(struct rte_eth_dev *dev, 460 struct rte_eth_xstat *xstats, 461 unsigned int size) 462 { 463 uint16_t i, j, count = gve_xstats_count(dev); 464 const char *stats; 465 466 if (xstats == NULL || size < count) 467 return count; 468 469 count = 0; 470 471 for (i = 0; i < dev->data->nb_tx_queues; i++) { 472 const struct gve_tx_queue *txq = dev->data->tx_queues[i]; 473 if (txq == NULL) 474 continue; 475 476 stats = (const char *)&txq->stats; 477 for (j = 0; j < RTE_DIM(tx_xstats_name_offset); j++, count++) { 478 xstats[count].id = count; 479 xstats[count].value = *(const uint64_t *) 480 (stats + tx_xstats_name_offset[j].offset); 481 } 482 } 483 484 for (i = 0; i < dev->data->nb_rx_queues; i++) { 485 const struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 486 if (rxq == NULL) 487 continue; 488 489 stats = (const char *)&rxq->stats; 490 for (j = 0; j < RTE_DIM(rx_xstats_name_offset); j++, count++) { 491 xstats[count].id = count; 492 xstats[count].value = *(const uint64_t *) 493 (stats + rx_xstats_name_offset[j].offset); 494 } 495 } 496 497 return count; 498 } 499 500 static int 501 gve_xstats_get_names(struct rte_eth_dev *dev, 502 struct rte_eth_xstat_name *xstats_names, 503 unsigned int size) 504 { 505 uint16_t i, j, count = gve_xstats_count(dev); 506 507 if (xstats_names == NULL || size < count) 508 return count; 509 510 count = 0; 511 512 for (i = 0; i < dev->data->nb_tx_queues; i++) { 513 if (dev->data->tx_queues[i] == NULL) 514 continue; 515 516 for (j = 0; j < RTE_DIM(tx_xstats_name_offset); j++) 517 snprintf(xstats_names[count++].name, 518 RTE_ETH_XSTATS_NAME_SIZE, 519 "tx_q%u_%s", i, tx_xstats_name_offset[j].name); 520 } 521 522 for (i = 0; i < dev->data->nb_rx_queues; i++) { 523 if (dev->data->rx_queues[i] == NULL) 524 continue; 525 526 for (j = 0; j < RTE_DIM(rx_xstats_name_offset); j++) 527 snprintf(xstats_names[count++].name, 528 RTE_ETH_XSTATS_NAME_SIZE, 529 "rx_q%u_%s", i, rx_xstats_name_offset[j].name); 530 } 531 532 return count; 533 } 534 535 static const struct eth_dev_ops gve_eth_dev_ops = { 536 .dev_configure = gve_dev_configure, 537 .dev_start = gve_dev_start, 538 .dev_stop = gve_dev_stop, 539 .dev_close = gve_dev_close, 540 .dev_infos_get = gve_dev_info_get, 541 .rx_queue_setup = gve_rx_queue_setup, 542 .tx_queue_setup = gve_tx_queue_setup, 543 .rx_queue_release = gve_rx_queue_release, 544 .tx_queue_release = gve_tx_queue_release, 545 .rx_queue_start = gve_rx_queue_start, 546 .tx_queue_start = gve_tx_queue_start, 547 .rx_queue_stop = gve_rx_queue_stop, 548 .tx_queue_stop = gve_tx_queue_stop, 549 .link_update = gve_link_update, 550 .stats_get = gve_dev_stats_get, 551 .stats_reset = gve_dev_stats_reset, 552 .mtu_set = gve_dev_mtu_set, 553 .xstats_get = gve_xstats_get, 554 .xstats_get_names = gve_xstats_get_names, 555 }; 556 557 static const struct eth_dev_ops gve_eth_dev_ops_dqo = { 558 .dev_configure = gve_dev_configure, 559 .dev_start = gve_dev_start, 560 .dev_stop = gve_dev_stop, 561 .dev_close = gve_dev_close, 562 .dev_infos_get = gve_dev_info_get, 563 .rx_queue_setup = gve_rx_queue_setup_dqo, 564 .tx_queue_setup = gve_tx_queue_setup_dqo, 565 .rx_queue_release = gve_rx_queue_release_dqo, 566 .tx_queue_release = gve_tx_queue_release_dqo, 567 .rx_queue_start = gve_rx_queue_start_dqo, 568 .tx_queue_start = gve_tx_queue_start_dqo, 569 .rx_queue_stop = gve_rx_queue_stop_dqo, 570 .tx_queue_stop = gve_tx_queue_stop_dqo, 571 .link_update = gve_link_update, 572 .stats_get = gve_dev_stats_get, 573 .stats_reset = gve_dev_stats_reset, 574 .mtu_set = gve_dev_mtu_set, 575 .xstats_get = gve_xstats_get, 576 .xstats_get_names = gve_xstats_get_names, 577 }; 578 579 static void 580 gve_free_counter_array(struct gve_priv *priv) 581 { 582 rte_memzone_free(priv->cnt_array_mz); 583 priv->cnt_array = NULL; 584 } 585 586 static void 587 gve_free_irq_db(struct gve_priv *priv) 588 { 589 rte_memzone_free(priv->irq_dbs_mz); 590 priv->irq_dbs = NULL; 591 } 592 593 static void 594 gve_teardown_device_resources(struct gve_priv *priv) 595 { 596 int err; 597 598 /* Tell device its resources are being freed */ 599 if (gve_get_device_resources_ok(priv)) { 600 err = gve_adminq_deconfigure_device_resources(priv); 601 if (err) 602 PMD_DRV_LOG(ERR, "Could not deconfigure device resources: err=%d", err); 603 } 604 gve_free_counter_array(priv); 605 gve_free_irq_db(priv); 606 gve_clear_device_resources_ok(priv); 607 } 608 609 static int 610 pci_dev_msix_vec_count(struct rte_pci_device *pdev) 611 { 612 off_t msix_pos = rte_pci_find_capability(pdev, RTE_PCI_CAP_ID_MSIX); 613 uint16_t control; 614 615 if (msix_pos > 0 && rte_pci_read_config(pdev, &control, sizeof(control), 616 msix_pos + RTE_PCI_MSIX_FLAGS) == sizeof(control)) 617 return (control & RTE_PCI_MSIX_FLAGS_QSIZE) + 1; 618 619 return 0; 620 } 621 622 static int 623 gve_setup_device_resources(struct gve_priv *priv) 624 { 625 char z_name[RTE_MEMZONE_NAMESIZE]; 626 const struct rte_memzone *mz; 627 int err = 0; 628 629 snprintf(z_name, sizeof(z_name), "gve_%s_cnt_arr", priv->pci_dev->device.name); 630 mz = rte_memzone_reserve_aligned(z_name, 631 priv->num_event_counters * sizeof(*priv->cnt_array), 632 rte_socket_id(), RTE_MEMZONE_IOVA_CONTIG, 633 PAGE_SIZE); 634 if (mz == NULL) { 635 PMD_DRV_LOG(ERR, "Could not alloc memzone for count array"); 636 return -ENOMEM; 637 } 638 priv->cnt_array = (rte_be32_t *)mz->addr; 639 priv->cnt_array_mz = mz; 640 641 snprintf(z_name, sizeof(z_name), "gve_%s_irqmz", priv->pci_dev->device.name); 642 mz = rte_memzone_reserve_aligned(z_name, 643 sizeof(*priv->irq_dbs) * (priv->num_ntfy_blks), 644 rte_socket_id(), RTE_MEMZONE_IOVA_CONTIG, 645 PAGE_SIZE); 646 if (mz == NULL) { 647 PMD_DRV_LOG(ERR, "Could not alloc memzone for irq_dbs"); 648 err = -ENOMEM; 649 goto free_cnt_array; 650 } 651 priv->irq_dbs = (struct gve_irq_db *)mz->addr; 652 priv->irq_dbs_mz = mz; 653 654 err = gve_adminq_configure_device_resources(priv, 655 priv->cnt_array_mz->iova, 656 priv->num_event_counters, 657 priv->irq_dbs_mz->iova, 658 priv->num_ntfy_blks); 659 if (unlikely(err)) { 660 PMD_DRV_LOG(ERR, "Could not config device resources: err=%d", err); 661 goto free_irq_dbs; 662 } 663 return 0; 664 665 free_irq_dbs: 666 gve_free_irq_db(priv); 667 free_cnt_array: 668 gve_free_counter_array(priv); 669 670 return err; 671 } 672 673 static int 674 gve_init_priv(struct gve_priv *priv, bool skip_describe_device) 675 { 676 uint16_t pages; 677 int num_ntfy; 678 uint32_t i; 679 int err; 680 681 /* Set up the adminq */ 682 err = gve_adminq_alloc(priv); 683 if (err) { 684 PMD_DRV_LOG(ERR, "Failed to alloc admin queue: err=%d", err); 685 return err; 686 } 687 err = gve_verify_driver_compatibility(priv); 688 if (err) { 689 PMD_DRV_LOG(ERR, "Could not verify driver compatibility: err=%d", err); 690 goto free_adminq; 691 } 692 693 if (skip_describe_device) 694 goto setup_device; 695 696 /* Get the initial information we need from the device */ 697 err = gve_adminq_describe_device(priv); 698 if (err) { 699 PMD_DRV_LOG(ERR, "Could not get device information: err=%d", err); 700 goto free_adminq; 701 } 702 703 num_ntfy = pci_dev_msix_vec_count(priv->pci_dev); 704 if (num_ntfy <= 0) { 705 PMD_DRV_LOG(ERR, "Could not count MSI-x vectors"); 706 err = -EIO; 707 goto free_adminq; 708 } else if (num_ntfy < GVE_MIN_MSIX) { 709 PMD_DRV_LOG(ERR, "GVE needs at least %d MSI-x vectors, but only has %d", 710 GVE_MIN_MSIX, num_ntfy); 711 err = -EINVAL; 712 goto free_adminq; 713 } 714 715 priv->num_registered_pages = 0; 716 717 /* gvnic has one Notification Block per MSI-x vector, except for the 718 * management vector 719 */ 720 priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1; 721 priv->mgmt_msix_idx = priv->num_ntfy_blks; 722 723 priv->max_nb_txq = RTE_MIN(priv->max_nb_txq, priv->num_ntfy_blks / 2); 724 priv->max_nb_rxq = RTE_MIN(priv->max_nb_rxq, priv->num_ntfy_blks / 2); 725 726 if (priv->default_num_queues > 0) { 727 priv->max_nb_txq = RTE_MIN(priv->default_num_queues, priv->max_nb_txq); 728 priv->max_nb_rxq = RTE_MIN(priv->default_num_queues, priv->max_nb_rxq); 729 } 730 731 PMD_DRV_LOG(INFO, "Max TX queues %d, Max RX queues %d", 732 priv->max_nb_txq, priv->max_nb_rxq); 733 734 /* In GQI_QPL queue format: 735 * Allocate queue page lists according to max queue number 736 * tx qpl id should start from 0 while rx qpl id should start 737 * from priv->max_nb_txq 738 */ 739 if (priv->queue_format == GVE_GQI_QPL_FORMAT) { 740 priv->qpl = rte_zmalloc("gve_qpl", 741 (priv->max_nb_txq + priv->max_nb_rxq) * 742 sizeof(struct gve_queue_page_list), 0); 743 if (priv->qpl == NULL) { 744 PMD_DRV_LOG(ERR, "Failed to alloc qpl."); 745 err = -ENOMEM; 746 goto free_adminq; 747 } 748 749 for (i = 0; i < priv->max_nb_txq + priv->max_nb_rxq; i++) { 750 if (i < priv->max_nb_txq) 751 pages = priv->tx_pages_per_qpl; 752 else 753 pages = priv->rx_data_slot_cnt; 754 err = gve_alloc_queue_page_list(priv, i, pages); 755 if (err != 0) { 756 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u.", i); 757 goto err_qpl; 758 } 759 } 760 } 761 762 setup_device: 763 err = gve_setup_device_resources(priv); 764 if (!err) 765 return 0; 766 err_qpl: 767 gve_free_qpls(priv); 768 free_adminq: 769 gve_adminq_free(priv); 770 return err; 771 } 772 773 static void 774 gve_teardown_priv_resources(struct gve_priv *priv) 775 { 776 gve_teardown_device_resources(priv); 777 gve_adminq_free(priv); 778 } 779 780 static int 781 gve_dev_init(struct rte_eth_dev *eth_dev) 782 { 783 struct gve_priv *priv = eth_dev->data->dev_private; 784 int max_tx_queues, max_rx_queues; 785 struct rte_pci_device *pci_dev; 786 struct gve_registers *reg_bar; 787 rte_be32_t *db_bar; 788 int err; 789 790 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 791 return 0; 792 793 pci_dev = RTE_DEV_TO_PCI(eth_dev->device); 794 795 reg_bar = pci_dev->mem_resource[GVE_REG_BAR].addr; 796 if (!reg_bar) { 797 PMD_DRV_LOG(ERR, "Failed to map pci bar!"); 798 return -ENOMEM; 799 } 800 801 db_bar = pci_dev->mem_resource[GVE_DB_BAR].addr; 802 if (!db_bar) { 803 PMD_DRV_LOG(ERR, "Failed to map doorbell bar!"); 804 return -ENOMEM; 805 } 806 807 gve_write_version(®_bar->driver_version); 808 /* Get max queues to alloc etherdev */ 809 max_tx_queues = ioread32be(®_bar->max_tx_queues); 810 max_rx_queues = ioread32be(®_bar->max_rx_queues); 811 812 priv->reg_bar0 = reg_bar; 813 priv->db_bar2 = db_bar; 814 priv->pci_dev = pci_dev; 815 priv->state_flags = 0x0; 816 817 priv->max_nb_txq = max_tx_queues; 818 priv->max_nb_rxq = max_rx_queues; 819 820 err = gve_init_priv(priv, false); 821 if (err) 822 return err; 823 824 if (gve_is_gqi(priv)) { 825 eth_dev->dev_ops = &gve_eth_dev_ops; 826 gve_set_rx_function(eth_dev); 827 gve_set_tx_function(eth_dev); 828 } else { 829 eth_dev->dev_ops = &gve_eth_dev_ops_dqo; 830 gve_set_rx_function_dqo(eth_dev); 831 gve_set_tx_function_dqo(eth_dev); 832 } 833 834 eth_dev->data->mac_addrs = &priv->dev_addr; 835 836 return 0; 837 } 838 839 static int 840 gve_dev_uninit(struct rte_eth_dev *eth_dev) 841 { 842 struct gve_priv *priv = eth_dev->data->dev_private; 843 844 gve_teardown_priv_resources(priv); 845 846 eth_dev->data->mac_addrs = NULL; 847 848 return 0; 849 } 850 851 static int 852 gve_pci_probe(__rte_unused struct rte_pci_driver *pci_drv, 853 struct rte_pci_device *pci_dev) 854 { 855 return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct gve_priv), gve_dev_init); 856 } 857 858 static int 859 gve_pci_remove(struct rte_pci_device *pci_dev) 860 { 861 return rte_eth_dev_pci_generic_remove(pci_dev, gve_dev_uninit); 862 } 863 864 static const struct rte_pci_id pci_id_gve_map[] = { 865 { RTE_PCI_DEVICE(GOOGLE_VENDOR_ID, GVE_DEV_ID) }, 866 { .device_id = 0 }, 867 }; 868 869 static struct rte_pci_driver rte_gve_pmd = { 870 .id_table = pci_id_gve_map, 871 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 872 .probe = gve_pci_probe, 873 .remove = gve_pci_remove, 874 }; 875 876 RTE_PMD_REGISTER_PCI(net_gve, rte_gve_pmd); 877 RTE_PMD_REGISTER_PCI_TABLE(net_gve, pci_id_gve_map); 878 RTE_PMD_REGISTER_KMOD_DEP(net_gve, "* igb_uio | vfio-pci"); 879 RTE_LOG_REGISTER_SUFFIX(gve_logtype_driver, driver, NOTICE); 880