1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2022 Intel Corporation 3 */ 4 5 #include "gve_ethdev.h" 6 #include "base/gve_adminq.h" 7 #include "base/gve_register.h" 8 #include "base/gve_osdep.h" 9 #include "gve_version.h" 10 #include "rte_ether.h" 11 12 static void 13 gve_write_version(uint8_t *driver_version_register) 14 { 15 const char *c = gve_version_string(); 16 while (*c) { 17 writeb(*c, driver_version_register); 18 c++; 19 } 20 writeb('\n', driver_version_register); 21 } 22 23 static int 24 gve_alloc_queue_page_list(struct gve_priv *priv, uint32_t id, uint32_t pages) 25 { 26 char z_name[RTE_MEMZONE_NAMESIZE]; 27 struct gve_queue_page_list *qpl; 28 const struct rte_memzone *mz; 29 dma_addr_t page_bus; 30 uint32_t i; 31 32 if (priv->num_registered_pages + pages > 33 priv->max_registered_pages) { 34 PMD_DRV_LOG(ERR, "Pages %" PRIu64 " > max registered pages %" PRIu64, 35 priv->num_registered_pages + pages, 36 priv->max_registered_pages); 37 return -EINVAL; 38 } 39 qpl = &priv->qpl[id]; 40 snprintf(z_name, sizeof(z_name), "gve_%s_qpl%d", priv->pci_dev->device.name, id); 41 mz = rte_memzone_reserve_aligned(z_name, pages * PAGE_SIZE, 42 rte_socket_id(), 43 RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); 44 if (mz == NULL) { 45 PMD_DRV_LOG(ERR, "Failed to alloc %s.", z_name); 46 return -ENOMEM; 47 } 48 qpl->page_buses = rte_zmalloc("qpl page buses", pages * sizeof(dma_addr_t), 0); 49 if (qpl->page_buses == NULL) { 50 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u page buses", id); 51 return -ENOMEM; 52 } 53 page_bus = mz->iova; 54 for (i = 0; i < pages; i++) { 55 qpl->page_buses[i] = page_bus; 56 page_bus += PAGE_SIZE; 57 } 58 qpl->id = id; 59 qpl->mz = mz; 60 qpl->num_entries = pages; 61 62 priv->num_registered_pages += pages; 63 64 return 0; 65 } 66 67 static void 68 gve_free_qpls(struct gve_priv *priv) 69 { 70 uint16_t nb_txqs = priv->max_nb_txq; 71 uint16_t nb_rxqs = priv->max_nb_rxq; 72 uint32_t i; 73 74 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 75 return; 76 77 for (i = 0; i < nb_txqs + nb_rxqs; i++) { 78 if (priv->qpl[i].mz != NULL) 79 rte_memzone_free(priv->qpl[i].mz); 80 rte_free(priv->qpl[i].page_buses); 81 } 82 83 rte_free(priv->qpl); 84 } 85 86 static int 87 gve_dev_configure(struct rte_eth_dev *dev) 88 { 89 struct gve_priv *priv = dev->data->dev_private; 90 91 if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) 92 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; 93 94 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) 95 priv->enable_rsc = 1; 96 97 return 0; 98 } 99 100 static int 101 gve_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 102 { 103 struct gve_priv *priv = dev->data->dev_private; 104 struct rte_eth_link link; 105 int err; 106 107 memset(&link, 0, sizeof(link)); 108 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 109 link.link_autoneg = RTE_ETH_LINK_AUTONEG; 110 111 if (!dev->data->dev_started) { 112 link.link_status = RTE_ETH_LINK_DOWN; 113 link.link_speed = RTE_ETH_SPEED_NUM_NONE; 114 } else { 115 link.link_status = RTE_ETH_LINK_UP; 116 PMD_DRV_LOG(DEBUG, "Get link status from hw"); 117 err = gve_adminq_report_link_speed(priv); 118 if (err) { 119 PMD_DRV_LOG(ERR, "Failed to get link speed."); 120 priv->link_speed = RTE_ETH_SPEED_NUM_UNKNOWN; 121 } 122 link.link_speed = priv->link_speed; 123 } 124 125 return rte_eth_linkstatus_set(dev, &link); 126 } 127 128 static int 129 gve_start_queues(struct rte_eth_dev *dev) 130 { 131 struct gve_priv *priv = dev->data->dev_private; 132 uint16_t num_queues; 133 uint16_t i; 134 int ret; 135 136 num_queues = dev->data->nb_tx_queues; 137 priv->txqs = (struct gve_tx_queue **)dev->data->tx_queues; 138 ret = gve_adminq_create_tx_queues(priv, num_queues); 139 if (ret != 0) { 140 PMD_DRV_LOG(ERR, "Failed to create %u tx queues.", num_queues); 141 return ret; 142 } 143 for (i = 0; i < num_queues; i++) 144 if (gve_tx_queue_start(dev, i) != 0) { 145 PMD_DRV_LOG(ERR, "Fail to start Tx queue %d", i); 146 goto err_tx; 147 } 148 149 num_queues = dev->data->nb_rx_queues; 150 priv->rxqs = (struct gve_rx_queue **)dev->data->rx_queues; 151 ret = gve_adminq_create_rx_queues(priv, num_queues); 152 if (ret != 0) { 153 PMD_DRV_LOG(ERR, "Failed to create %u rx queues.", num_queues); 154 goto err_tx; 155 } 156 for (i = 0; i < num_queues; i++) { 157 if (gve_is_gqi(priv)) 158 ret = gve_rx_queue_start(dev, i); 159 else 160 ret = gve_rx_queue_start_dqo(dev, i); 161 if (ret != 0) { 162 PMD_DRV_LOG(ERR, "Fail to start Rx queue %d", i); 163 goto err_rx; 164 } 165 } 166 167 return 0; 168 169 err_rx: 170 gve_stop_rx_queues(dev); 171 err_tx: 172 gve_stop_tx_queues(dev); 173 return ret; 174 } 175 176 static int 177 gve_dev_start(struct rte_eth_dev *dev) 178 { 179 int ret; 180 181 ret = gve_start_queues(dev); 182 if (ret != 0) { 183 PMD_DRV_LOG(ERR, "Failed to start queues"); 184 return ret; 185 } 186 187 dev->data->dev_started = 1; 188 gve_link_update(dev, 0); 189 190 return 0; 191 } 192 193 static int 194 gve_dev_stop(struct rte_eth_dev *dev) 195 { 196 dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; 197 198 gve_stop_tx_queues(dev); 199 gve_stop_rx_queues(dev); 200 201 dev->data->dev_started = 0; 202 203 return 0; 204 } 205 206 static int 207 gve_dev_close(struct rte_eth_dev *dev) 208 { 209 struct gve_priv *priv = dev->data->dev_private; 210 int err = 0; 211 uint16_t i; 212 213 if (dev->data->dev_started) { 214 err = gve_dev_stop(dev); 215 if (err != 0) 216 PMD_DRV_LOG(ERR, "Failed to stop dev."); 217 } 218 219 if (gve_is_gqi(priv)) { 220 for (i = 0; i < dev->data->nb_tx_queues; i++) 221 gve_tx_queue_release(dev, i); 222 223 for (i = 0; i < dev->data->nb_rx_queues; i++) 224 gve_rx_queue_release(dev, i); 225 } else { 226 for (i = 0; i < dev->data->nb_tx_queues; i++) 227 gve_tx_queue_release_dqo(dev, i); 228 229 for (i = 0; i < dev->data->nb_rx_queues; i++) 230 gve_rx_queue_release_dqo(dev, i); 231 } 232 233 gve_free_qpls(priv); 234 rte_free(priv->adminq); 235 236 dev->data->mac_addrs = NULL; 237 238 return err; 239 } 240 241 static int 242 gve_verify_driver_compatibility(struct gve_priv *priv) 243 { 244 const struct rte_memzone *driver_info_mem; 245 struct gve_driver_info *driver_info; 246 int err; 247 248 driver_info_mem = rte_memzone_reserve_aligned("verify_driver_compatibility", 249 sizeof(struct gve_driver_info), 250 rte_socket_id(), 251 RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); 252 253 if (driver_info_mem == NULL) { 254 PMD_DRV_LOG(ERR, 255 "Could not alloc memzone for driver compatibility"); 256 return -ENOMEM; 257 } 258 driver_info = (struct gve_driver_info *)driver_info_mem->addr; 259 260 *driver_info = (struct gve_driver_info) { 261 .os_type = 5, /* DPDK */ 262 .driver_major = GVE_VERSION_MAJOR, 263 .driver_minor = GVE_VERSION_MINOR, 264 .driver_sub = GVE_VERSION_SUB, 265 .os_version_major = cpu_to_be32(DPDK_VERSION_MAJOR), 266 .os_version_minor = cpu_to_be32(DPDK_VERSION_MINOR), 267 .os_version_sub = cpu_to_be32(DPDK_VERSION_SUB), 268 .driver_capability_flags = { 269 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS1), 270 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS2), 271 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS3), 272 cpu_to_be64(GVE_DRIVER_CAPABILITY_FLAGS4), 273 }, 274 }; 275 276 populate_driver_version_strings((char *)driver_info->os_version_str1, 277 (char *)driver_info->os_version_str2); 278 279 err = gve_adminq_verify_driver_compatibility(priv, 280 sizeof(struct gve_driver_info), 281 (dma_addr_t)driver_info_mem->iova); 282 /* It's ok if the device doesn't support this */ 283 if (err == -EOPNOTSUPP) 284 err = 0; 285 286 rte_memzone_free(driver_info_mem); 287 return err; 288 } 289 290 static int 291 gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 292 { 293 struct gve_priv *priv = dev->data->dev_private; 294 295 dev_info->device = dev->device; 296 dev_info->max_mac_addrs = 1; 297 dev_info->max_rx_queues = priv->max_nb_rxq; 298 dev_info->max_tx_queues = priv->max_nb_txq; 299 if (gve_is_gqi(priv)) { 300 dev_info->min_rx_bufsize = GVE_RX_MIN_BUF_SIZE_GQI; 301 dev_info->max_rx_bufsize = GVE_RX_MAX_BUF_SIZE_GQI; 302 } else { 303 dev_info->min_rx_bufsize = GVE_RX_MIN_BUF_SIZE_DQO; 304 dev_info->max_rx_bufsize = GVE_RX_MAX_BUF_SIZE_DQO; 305 } 306 307 dev_info->max_rx_pktlen = priv->max_mtu + RTE_ETHER_HDR_LEN; 308 dev_info->max_mtu = priv->max_mtu; 309 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 310 311 dev_info->rx_offload_capa = 0; 312 dev_info->tx_offload_capa = 313 RTE_ETH_TX_OFFLOAD_MULTI_SEGS | 314 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 315 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 316 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 317 RTE_ETH_TX_OFFLOAD_TCP_TSO; 318 319 if (priv->queue_format == GVE_DQO_RDA_FORMAT) 320 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO; 321 322 dev_info->default_rxconf = (struct rte_eth_rxconf) { 323 .rx_free_thresh = GVE_DEFAULT_RX_FREE_THRESH, 324 .rx_drop_en = 0, 325 .offloads = 0, 326 }; 327 328 dev_info->default_txconf = (struct rte_eth_txconf) { 329 .tx_free_thresh = GVE_DEFAULT_TX_FREE_THRESH, 330 .tx_rs_thresh = GVE_DEFAULT_TX_RS_THRESH, 331 .offloads = 0, 332 }; 333 334 dev_info->default_rxportconf.ring_size = priv->rx_desc_cnt; 335 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 336 .nb_max = gve_is_gqi(priv) ? priv->rx_desc_cnt : GVE_MAX_QUEUE_SIZE_DQO, 337 .nb_min = priv->rx_desc_cnt, 338 .nb_align = 1, 339 }; 340 341 dev_info->default_txportconf.ring_size = priv->tx_desc_cnt; 342 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 343 .nb_max = gve_is_gqi(priv) ? priv->tx_desc_cnt : GVE_MAX_QUEUE_SIZE_DQO, 344 .nb_min = priv->tx_desc_cnt, 345 .nb_align = 1, 346 }; 347 348 return 0; 349 } 350 351 static int 352 gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 353 { 354 uint16_t i; 355 356 for (i = 0; i < dev->data->nb_tx_queues; i++) { 357 struct gve_tx_queue *txq = dev->data->tx_queues[i]; 358 if (txq == NULL) 359 continue; 360 361 stats->opackets += txq->stats.packets; 362 stats->obytes += txq->stats.bytes; 363 stats->oerrors += txq->stats.errors; 364 } 365 366 for (i = 0; i < dev->data->nb_rx_queues; i++) { 367 struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 368 if (rxq == NULL) 369 continue; 370 371 stats->ipackets += rxq->stats.packets; 372 stats->ibytes += rxq->stats.bytes; 373 stats->ierrors += rxq->stats.errors; 374 stats->rx_nombuf += rxq->stats.no_mbufs; 375 } 376 377 return 0; 378 } 379 380 static int 381 gve_dev_stats_reset(struct rte_eth_dev *dev) 382 { 383 uint16_t i; 384 385 for (i = 0; i < dev->data->nb_tx_queues; i++) { 386 struct gve_tx_queue *txq = dev->data->tx_queues[i]; 387 if (txq == NULL) 388 continue; 389 390 memset(&txq->stats, 0, sizeof(txq->stats)); 391 } 392 393 for (i = 0; i < dev->data->nb_rx_queues; i++) { 394 struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 395 if (rxq == NULL) 396 continue; 397 398 memset(&rxq->stats, 0, sizeof(rxq->stats)); 399 } 400 401 return 0; 402 } 403 404 static int 405 gve_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 406 { 407 struct gve_priv *priv = dev->data->dev_private; 408 int err; 409 410 if (mtu < RTE_ETHER_MIN_MTU || mtu > priv->max_mtu) { 411 PMD_DRV_LOG(ERR, "MIN MTU is %u, MAX MTU is %u", 412 RTE_ETHER_MIN_MTU, priv->max_mtu); 413 return -EINVAL; 414 } 415 416 /* mtu setting is forbidden if port is start */ 417 if (dev->data->dev_started) { 418 PMD_DRV_LOG(ERR, "Port must be stopped before configuration"); 419 return -EBUSY; 420 } 421 422 err = gve_adminq_set_mtu(priv, mtu); 423 if (err) { 424 PMD_DRV_LOG(ERR, "Failed to set mtu as %u err = %d", mtu, err); 425 return err; 426 } 427 428 return 0; 429 } 430 431 #define TX_QUEUE_STATS_OFFSET(x) offsetof(struct gve_tx_stats, x) 432 #define RX_QUEUE_STATS_OFFSET(x) offsetof(struct gve_rx_stats, x) 433 434 static const struct gve_xstats_name_offset tx_xstats_name_offset[] = { 435 { "packets", TX_QUEUE_STATS_OFFSET(packets) }, 436 { "bytes", TX_QUEUE_STATS_OFFSET(bytes) }, 437 { "errors", TX_QUEUE_STATS_OFFSET(errors) }, 438 }; 439 440 static const struct gve_xstats_name_offset rx_xstats_name_offset[] = { 441 { "packets", RX_QUEUE_STATS_OFFSET(packets) }, 442 { "bytes", RX_QUEUE_STATS_OFFSET(bytes) }, 443 { "errors", RX_QUEUE_STATS_OFFSET(errors) }, 444 { "mbuf_alloc_errors", RX_QUEUE_STATS_OFFSET(no_mbufs) }, 445 { "mbuf_alloc_errors_bulk", RX_QUEUE_STATS_OFFSET(no_mbufs_bulk) }, 446 }; 447 448 static int 449 gve_xstats_count(struct rte_eth_dev *dev) 450 { 451 uint16_t i, count = 0; 452 453 for (i = 0; i < dev->data->nb_tx_queues; i++) { 454 if (dev->data->tx_queues[i]) 455 count += RTE_DIM(tx_xstats_name_offset); 456 } 457 458 for (i = 0; i < dev->data->nb_rx_queues; i++) { 459 if (dev->data->rx_queues[i]) 460 count += RTE_DIM(rx_xstats_name_offset); 461 } 462 463 return count; 464 } 465 466 static int 467 gve_xstats_get(struct rte_eth_dev *dev, 468 struct rte_eth_xstat *xstats, 469 unsigned int size) 470 { 471 uint16_t i, j, count = gve_xstats_count(dev); 472 const char *stats; 473 474 if (xstats == NULL || size < count) 475 return count; 476 477 count = 0; 478 479 for (i = 0; i < dev->data->nb_tx_queues; i++) { 480 const struct gve_tx_queue *txq = dev->data->tx_queues[i]; 481 if (txq == NULL) 482 continue; 483 484 stats = (const char *)&txq->stats; 485 for (j = 0; j < RTE_DIM(tx_xstats_name_offset); j++, count++) { 486 xstats[count].id = count; 487 xstats[count].value = *(const uint64_t *) 488 (stats + tx_xstats_name_offset[j].offset); 489 } 490 } 491 492 for (i = 0; i < dev->data->nb_rx_queues; i++) { 493 const struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 494 if (rxq == NULL) 495 continue; 496 497 stats = (const char *)&rxq->stats; 498 for (j = 0; j < RTE_DIM(rx_xstats_name_offset); j++, count++) { 499 xstats[count].id = count; 500 xstats[count].value = *(const uint64_t *) 501 (stats + rx_xstats_name_offset[j].offset); 502 } 503 } 504 505 return count; 506 } 507 508 static int 509 gve_xstats_get_names(struct rte_eth_dev *dev, 510 struct rte_eth_xstat_name *xstats_names, 511 unsigned int size) 512 { 513 uint16_t i, j, count = gve_xstats_count(dev); 514 515 if (xstats_names == NULL || size < count) 516 return count; 517 518 count = 0; 519 520 for (i = 0; i < dev->data->nb_tx_queues; i++) { 521 if (dev->data->tx_queues[i] == NULL) 522 continue; 523 524 for (j = 0; j < RTE_DIM(tx_xstats_name_offset); j++) 525 snprintf(xstats_names[count++].name, 526 RTE_ETH_XSTATS_NAME_SIZE, 527 "tx_q%u_%s", i, tx_xstats_name_offset[j].name); 528 } 529 530 for (i = 0; i < dev->data->nb_rx_queues; i++) { 531 if (dev->data->rx_queues[i] == NULL) 532 continue; 533 534 for (j = 0; j < RTE_DIM(rx_xstats_name_offset); j++) 535 snprintf(xstats_names[count++].name, 536 RTE_ETH_XSTATS_NAME_SIZE, 537 "rx_q%u_%s", i, rx_xstats_name_offset[j].name); 538 } 539 540 return count; 541 } 542 543 static const struct eth_dev_ops gve_eth_dev_ops = { 544 .dev_configure = gve_dev_configure, 545 .dev_start = gve_dev_start, 546 .dev_stop = gve_dev_stop, 547 .dev_close = gve_dev_close, 548 .dev_infos_get = gve_dev_info_get, 549 .rx_queue_setup = gve_rx_queue_setup, 550 .tx_queue_setup = gve_tx_queue_setup, 551 .rx_queue_release = gve_rx_queue_release, 552 .tx_queue_release = gve_tx_queue_release, 553 .rx_queue_start = gve_rx_queue_start, 554 .tx_queue_start = gve_tx_queue_start, 555 .rx_queue_stop = gve_rx_queue_stop, 556 .tx_queue_stop = gve_tx_queue_stop, 557 .link_update = gve_link_update, 558 .stats_get = gve_dev_stats_get, 559 .stats_reset = gve_dev_stats_reset, 560 .mtu_set = gve_dev_mtu_set, 561 .xstats_get = gve_xstats_get, 562 .xstats_get_names = gve_xstats_get_names, 563 }; 564 565 static const struct eth_dev_ops gve_eth_dev_ops_dqo = { 566 .dev_configure = gve_dev_configure, 567 .dev_start = gve_dev_start, 568 .dev_stop = gve_dev_stop, 569 .dev_close = gve_dev_close, 570 .dev_infos_get = gve_dev_info_get, 571 .rx_queue_setup = gve_rx_queue_setup_dqo, 572 .tx_queue_setup = gve_tx_queue_setup_dqo, 573 .rx_queue_release = gve_rx_queue_release_dqo, 574 .tx_queue_release = gve_tx_queue_release_dqo, 575 .rx_queue_start = gve_rx_queue_start_dqo, 576 .tx_queue_start = gve_tx_queue_start_dqo, 577 .rx_queue_stop = gve_rx_queue_stop_dqo, 578 .tx_queue_stop = gve_tx_queue_stop_dqo, 579 .link_update = gve_link_update, 580 .stats_get = gve_dev_stats_get, 581 .stats_reset = gve_dev_stats_reset, 582 .mtu_set = gve_dev_mtu_set, 583 .xstats_get = gve_xstats_get, 584 .xstats_get_names = gve_xstats_get_names, 585 }; 586 587 static void 588 gve_free_counter_array(struct gve_priv *priv) 589 { 590 rte_memzone_free(priv->cnt_array_mz); 591 priv->cnt_array = NULL; 592 } 593 594 static void 595 gve_free_irq_db(struct gve_priv *priv) 596 { 597 rte_memzone_free(priv->irq_dbs_mz); 598 priv->irq_dbs = NULL; 599 } 600 601 static void 602 gve_teardown_device_resources(struct gve_priv *priv) 603 { 604 int err; 605 606 /* Tell device its resources are being freed */ 607 if (gve_get_device_resources_ok(priv)) { 608 err = gve_adminq_deconfigure_device_resources(priv); 609 if (err) 610 PMD_DRV_LOG(ERR, "Could not deconfigure device resources: err=%d", err); 611 } 612 gve_free_counter_array(priv); 613 gve_free_irq_db(priv); 614 gve_clear_device_resources_ok(priv); 615 } 616 617 static int 618 pci_dev_msix_vec_count(struct rte_pci_device *pdev) 619 { 620 off_t msix_pos = rte_pci_find_capability(pdev, RTE_PCI_CAP_ID_MSIX); 621 uint16_t control; 622 623 if (msix_pos > 0 && rte_pci_read_config(pdev, &control, sizeof(control), 624 msix_pos + RTE_PCI_MSIX_FLAGS) == sizeof(control)) 625 return (control & RTE_PCI_MSIX_FLAGS_QSIZE) + 1; 626 627 return 0; 628 } 629 630 static int 631 gve_setup_device_resources(struct gve_priv *priv) 632 { 633 char z_name[RTE_MEMZONE_NAMESIZE]; 634 const struct rte_memzone *mz; 635 int err = 0; 636 637 snprintf(z_name, sizeof(z_name), "gve_%s_cnt_arr", priv->pci_dev->device.name); 638 mz = rte_memzone_reserve_aligned(z_name, 639 priv->num_event_counters * sizeof(*priv->cnt_array), 640 rte_socket_id(), RTE_MEMZONE_IOVA_CONTIG, 641 PAGE_SIZE); 642 if (mz == NULL) { 643 PMD_DRV_LOG(ERR, "Could not alloc memzone for count array"); 644 return -ENOMEM; 645 } 646 priv->cnt_array = (rte_be32_t *)mz->addr; 647 priv->cnt_array_mz = mz; 648 649 snprintf(z_name, sizeof(z_name), "gve_%s_irqmz", priv->pci_dev->device.name); 650 mz = rte_memzone_reserve_aligned(z_name, 651 sizeof(*priv->irq_dbs) * (priv->num_ntfy_blks), 652 rte_socket_id(), RTE_MEMZONE_IOVA_CONTIG, 653 PAGE_SIZE); 654 if (mz == NULL) { 655 PMD_DRV_LOG(ERR, "Could not alloc memzone for irq_dbs"); 656 err = -ENOMEM; 657 goto free_cnt_array; 658 } 659 priv->irq_dbs = (struct gve_irq_db *)mz->addr; 660 priv->irq_dbs_mz = mz; 661 662 err = gve_adminq_configure_device_resources(priv, 663 priv->cnt_array_mz->iova, 664 priv->num_event_counters, 665 priv->irq_dbs_mz->iova, 666 priv->num_ntfy_blks); 667 if (unlikely(err)) { 668 PMD_DRV_LOG(ERR, "Could not config device resources: err=%d", err); 669 goto free_irq_dbs; 670 } 671 return 0; 672 673 free_irq_dbs: 674 gve_free_irq_db(priv); 675 free_cnt_array: 676 gve_free_counter_array(priv); 677 678 return err; 679 } 680 681 static int 682 gve_init_priv(struct gve_priv *priv, bool skip_describe_device) 683 { 684 uint16_t pages; 685 int num_ntfy; 686 uint32_t i; 687 int err; 688 689 /* Set up the adminq */ 690 err = gve_adminq_alloc(priv); 691 if (err) { 692 PMD_DRV_LOG(ERR, "Failed to alloc admin queue: err=%d", err); 693 return err; 694 } 695 err = gve_verify_driver_compatibility(priv); 696 if (err) { 697 PMD_DRV_LOG(ERR, "Could not verify driver compatibility: err=%d", err); 698 goto free_adminq; 699 } 700 701 if (skip_describe_device) 702 goto setup_device; 703 704 /* Get the initial information we need from the device */ 705 err = gve_adminq_describe_device(priv); 706 if (err) { 707 PMD_DRV_LOG(ERR, "Could not get device information: err=%d", err); 708 goto free_adminq; 709 } 710 711 num_ntfy = pci_dev_msix_vec_count(priv->pci_dev); 712 if (num_ntfy <= 0) { 713 PMD_DRV_LOG(ERR, "Could not count MSI-x vectors"); 714 err = -EIO; 715 goto free_adminq; 716 } else if (num_ntfy < GVE_MIN_MSIX) { 717 PMD_DRV_LOG(ERR, "GVE needs at least %d MSI-x vectors, but only has %d", 718 GVE_MIN_MSIX, num_ntfy); 719 err = -EINVAL; 720 goto free_adminq; 721 } 722 723 priv->num_registered_pages = 0; 724 725 /* gvnic has one Notification Block per MSI-x vector, except for the 726 * management vector 727 */ 728 priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1; 729 priv->mgmt_msix_idx = priv->num_ntfy_blks; 730 731 priv->max_nb_txq = RTE_MIN(priv->max_nb_txq, priv->num_ntfy_blks / 2); 732 priv->max_nb_rxq = RTE_MIN(priv->max_nb_rxq, priv->num_ntfy_blks / 2); 733 734 if (priv->default_num_queues > 0) { 735 priv->max_nb_txq = RTE_MIN(priv->default_num_queues, priv->max_nb_txq); 736 priv->max_nb_rxq = RTE_MIN(priv->default_num_queues, priv->max_nb_rxq); 737 } 738 739 PMD_DRV_LOG(INFO, "Max TX queues %d, Max RX queues %d", 740 priv->max_nb_txq, priv->max_nb_rxq); 741 742 /* In GQI_QPL queue format: 743 * Allocate queue page lists according to max queue number 744 * tx qpl id should start from 0 while rx qpl id should start 745 * from priv->max_nb_txq 746 */ 747 if (priv->queue_format == GVE_GQI_QPL_FORMAT) { 748 priv->qpl = rte_zmalloc("gve_qpl", 749 (priv->max_nb_txq + priv->max_nb_rxq) * 750 sizeof(struct gve_queue_page_list), 0); 751 if (priv->qpl == NULL) { 752 PMD_DRV_LOG(ERR, "Failed to alloc qpl."); 753 err = -ENOMEM; 754 goto free_adminq; 755 } 756 757 for (i = 0; i < priv->max_nb_txq + priv->max_nb_rxq; i++) { 758 if (i < priv->max_nb_txq) 759 pages = priv->tx_pages_per_qpl; 760 else 761 pages = priv->rx_data_slot_cnt; 762 err = gve_alloc_queue_page_list(priv, i, pages); 763 if (err != 0) { 764 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u.", i); 765 goto err_qpl; 766 } 767 } 768 } 769 770 setup_device: 771 err = gve_setup_device_resources(priv); 772 if (!err) 773 return 0; 774 err_qpl: 775 gve_free_qpls(priv); 776 free_adminq: 777 gve_adminq_free(priv); 778 return err; 779 } 780 781 static void 782 gve_teardown_priv_resources(struct gve_priv *priv) 783 { 784 gve_teardown_device_resources(priv); 785 gve_adminq_free(priv); 786 } 787 788 static int 789 gve_dev_init(struct rte_eth_dev *eth_dev) 790 { 791 struct gve_priv *priv = eth_dev->data->dev_private; 792 int max_tx_queues, max_rx_queues; 793 struct rte_pci_device *pci_dev; 794 struct gve_registers *reg_bar; 795 rte_be32_t *db_bar; 796 int err; 797 798 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 799 return 0; 800 801 pci_dev = RTE_DEV_TO_PCI(eth_dev->device); 802 803 reg_bar = pci_dev->mem_resource[GVE_REG_BAR].addr; 804 if (!reg_bar) { 805 PMD_DRV_LOG(ERR, "Failed to map pci bar!"); 806 return -ENOMEM; 807 } 808 809 db_bar = pci_dev->mem_resource[GVE_DB_BAR].addr; 810 if (!db_bar) { 811 PMD_DRV_LOG(ERR, "Failed to map doorbell bar!"); 812 return -ENOMEM; 813 } 814 815 gve_write_version(®_bar->driver_version); 816 /* Get max queues to alloc etherdev */ 817 max_tx_queues = ioread32be(®_bar->max_tx_queues); 818 max_rx_queues = ioread32be(®_bar->max_rx_queues); 819 820 priv->reg_bar0 = reg_bar; 821 priv->db_bar2 = db_bar; 822 priv->pci_dev = pci_dev; 823 priv->state_flags = 0x0; 824 825 priv->max_nb_txq = max_tx_queues; 826 priv->max_nb_rxq = max_rx_queues; 827 828 err = gve_init_priv(priv, false); 829 if (err) 830 return err; 831 832 if (gve_is_gqi(priv)) { 833 eth_dev->dev_ops = &gve_eth_dev_ops; 834 gve_set_rx_function(eth_dev); 835 gve_set_tx_function(eth_dev); 836 } else { 837 eth_dev->dev_ops = &gve_eth_dev_ops_dqo; 838 gve_set_rx_function_dqo(eth_dev); 839 gve_set_tx_function_dqo(eth_dev); 840 } 841 842 eth_dev->data->mac_addrs = &priv->dev_addr; 843 844 return 0; 845 } 846 847 static int 848 gve_dev_uninit(struct rte_eth_dev *eth_dev) 849 { 850 struct gve_priv *priv = eth_dev->data->dev_private; 851 852 gve_teardown_priv_resources(priv); 853 854 eth_dev->data->mac_addrs = NULL; 855 856 return 0; 857 } 858 859 static int 860 gve_pci_probe(__rte_unused struct rte_pci_driver *pci_drv, 861 struct rte_pci_device *pci_dev) 862 { 863 return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct gve_priv), gve_dev_init); 864 } 865 866 static int 867 gve_pci_remove(struct rte_pci_device *pci_dev) 868 { 869 return rte_eth_dev_pci_generic_remove(pci_dev, gve_dev_uninit); 870 } 871 872 static const struct rte_pci_id pci_id_gve_map[] = { 873 { RTE_PCI_DEVICE(GOOGLE_VENDOR_ID, GVE_DEV_ID) }, 874 { .device_id = 0 }, 875 }; 876 877 static struct rte_pci_driver rte_gve_pmd = { 878 .id_table = pci_id_gve_map, 879 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 880 .probe = gve_pci_probe, 881 .remove = gve_pci_remove, 882 }; 883 884 RTE_PMD_REGISTER_PCI(net_gve, rte_gve_pmd); 885 RTE_PMD_REGISTER_PCI_TABLE(net_gve, pci_id_gve_map); 886 RTE_PMD_REGISTER_KMOD_DEP(net_gve, "* igb_uio | vfio-pci"); 887 RTE_LOG_REGISTER_SUFFIX(gve_logtype_driver, driver, NOTICE); 888