1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2022 Intel Corporation 3 */ 4 5 #include "gve_ethdev.h" 6 #include "base/gve_adminq.h" 7 #include "base/gve_register.h" 8 9 const char gve_version_str[] = GVE_VERSION; 10 static const char gve_version_prefix[] = GVE_VERSION_PREFIX; 11 12 static void 13 gve_write_version(uint8_t *driver_version_register) 14 { 15 const char *c = gve_version_prefix; 16 17 while (*c) { 18 writeb(*c, driver_version_register); 19 c++; 20 } 21 22 c = gve_version_str; 23 while (*c) { 24 writeb(*c, driver_version_register); 25 c++; 26 } 27 writeb('\n', driver_version_register); 28 } 29 30 static int 31 gve_alloc_queue_page_list(struct gve_priv *priv, uint32_t id, uint32_t pages) 32 { 33 char z_name[RTE_MEMZONE_NAMESIZE]; 34 struct gve_queue_page_list *qpl; 35 const struct rte_memzone *mz; 36 dma_addr_t page_bus; 37 uint32_t i; 38 39 if (priv->num_registered_pages + pages > 40 priv->max_registered_pages) { 41 PMD_DRV_LOG(ERR, "Pages %" PRIu64 " > max registered pages %" PRIu64, 42 priv->num_registered_pages + pages, 43 priv->max_registered_pages); 44 return -EINVAL; 45 } 46 qpl = &priv->qpl[id]; 47 snprintf(z_name, sizeof(z_name), "gve_%s_qpl%d", priv->pci_dev->device.name, id); 48 mz = rte_memzone_reserve_aligned(z_name, pages * PAGE_SIZE, 49 rte_socket_id(), 50 RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); 51 if (mz == NULL) { 52 PMD_DRV_LOG(ERR, "Failed to alloc %s.", z_name); 53 return -ENOMEM; 54 } 55 qpl->page_buses = rte_zmalloc("qpl page buses", pages * sizeof(dma_addr_t), 0); 56 if (qpl->page_buses == NULL) { 57 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u page buses", id); 58 return -ENOMEM; 59 } 60 page_bus = mz->iova; 61 for (i = 0; i < pages; i++) { 62 qpl->page_buses[i] = page_bus; 63 page_bus += PAGE_SIZE; 64 } 65 qpl->id = id; 66 qpl->mz = mz; 67 qpl->num_entries = pages; 68 69 priv->num_registered_pages += pages; 70 71 return 0; 72 } 73 74 static void 75 gve_free_qpls(struct gve_priv *priv) 76 { 77 uint16_t nb_txqs = priv->max_nb_txq; 78 uint16_t nb_rxqs = priv->max_nb_rxq; 79 uint32_t i; 80 81 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 82 return; 83 84 for (i = 0; i < nb_txqs + nb_rxqs; i++) { 85 if (priv->qpl[i].mz != NULL) 86 rte_memzone_free(priv->qpl[i].mz); 87 rte_free(priv->qpl[i].page_buses); 88 } 89 90 rte_free(priv->qpl); 91 } 92 93 static int 94 gve_dev_configure(struct rte_eth_dev *dev) 95 { 96 struct gve_priv *priv = dev->data->dev_private; 97 98 if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) 99 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; 100 101 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) 102 priv->enable_rsc = 1; 103 104 return 0; 105 } 106 107 static int 108 gve_refill_pages(struct gve_rx_queue *rxq) 109 { 110 struct rte_mbuf *nmb; 111 uint16_t i; 112 int diag; 113 114 diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[0], rxq->nb_rx_desc); 115 if (diag < 0) { 116 for (i = 0; i < rxq->nb_rx_desc - 1; i++) { 117 nmb = rte_pktmbuf_alloc(rxq->mpool); 118 if (!nmb) 119 break; 120 rxq->sw_ring[i] = nmb; 121 } 122 if (i < rxq->nb_rx_desc - 1) 123 return -ENOMEM; 124 } 125 rxq->nb_avail = 0; 126 rxq->next_avail = rxq->nb_rx_desc - 1; 127 128 for (i = 0; i < rxq->nb_rx_desc; i++) { 129 if (rxq->is_gqi_qpl) { 130 rxq->rx_data_ring[i].addr = rte_cpu_to_be_64(i * PAGE_SIZE); 131 } else { 132 if (i == rxq->nb_rx_desc - 1) 133 break; 134 nmb = rxq->sw_ring[i]; 135 rxq->rx_data_ring[i].addr = rte_cpu_to_be_64(rte_mbuf_data_iova(nmb)); 136 } 137 } 138 139 rte_write32(rte_cpu_to_be_32(rxq->next_avail), rxq->qrx_tail); 140 141 return 0; 142 } 143 144 static int 145 gve_refill_dqo(struct gve_rx_queue *rxq) 146 { 147 struct rte_mbuf *nmb; 148 uint16_t i; 149 int diag; 150 151 diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[0], rxq->nb_rx_desc); 152 if (diag < 0) { 153 for (i = 0; i < rxq->nb_rx_desc - 1; i++) { 154 nmb = rte_pktmbuf_alloc(rxq->mpool); 155 if (!nmb) 156 break; 157 rxq->sw_ring[i] = nmb; 158 } 159 if (i < rxq->nb_rx_desc - 1) 160 return -ENOMEM; 161 } 162 163 for (i = 0; i < rxq->nb_rx_desc; i++) { 164 if (i == rxq->nb_rx_desc - 1) 165 break; 166 nmb = rxq->sw_ring[i]; 167 rxq->rx_ring[i].buf_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 168 rxq->rx_ring[i].buf_id = rte_cpu_to_le_16(i); 169 } 170 171 rxq->nb_rx_hold = 0; 172 rxq->bufq_tail = rxq->nb_rx_desc - 1; 173 174 rte_write32(rxq->bufq_tail, rxq->qrx_tail); 175 176 return 0; 177 } 178 179 static int 180 gve_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 181 { 182 struct gve_priv *priv = dev->data->dev_private; 183 struct rte_eth_link link; 184 int err; 185 186 memset(&link, 0, sizeof(link)); 187 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 188 link.link_autoneg = RTE_ETH_LINK_AUTONEG; 189 190 if (!dev->data->dev_started) { 191 link.link_status = RTE_ETH_LINK_DOWN; 192 link.link_speed = RTE_ETH_SPEED_NUM_NONE; 193 } else { 194 link.link_status = RTE_ETH_LINK_UP; 195 PMD_DRV_LOG(DEBUG, "Get link status from hw"); 196 err = gve_adminq_report_link_speed(priv); 197 if (err) { 198 PMD_DRV_LOG(ERR, "Failed to get link speed."); 199 priv->link_speed = RTE_ETH_SPEED_NUM_UNKNOWN; 200 } 201 link.link_speed = priv->link_speed; 202 } 203 204 return rte_eth_linkstatus_set(dev, &link); 205 } 206 207 static int 208 gve_dev_start(struct rte_eth_dev *dev) 209 { 210 uint16_t num_queues = dev->data->nb_tx_queues; 211 struct gve_priv *priv = dev->data->dev_private; 212 struct gve_tx_queue *txq; 213 struct gve_rx_queue *rxq; 214 uint16_t i; 215 int err; 216 217 priv->txqs = (struct gve_tx_queue **)dev->data->tx_queues; 218 err = gve_adminq_create_tx_queues(priv, num_queues); 219 if (err) { 220 PMD_DRV_LOG(ERR, "failed to create %u tx queues.", num_queues); 221 return err; 222 } 223 for (i = 0; i < num_queues; i++) { 224 txq = priv->txqs[i]; 225 txq->qtx_tail = 226 &priv->db_bar2[rte_be_to_cpu_32(txq->qres->db_index)]; 227 txq->qtx_head = 228 &priv->cnt_array[rte_be_to_cpu_32(txq->qres->counter_index)]; 229 230 rte_write32(rte_cpu_to_be_32(GVE_IRQ_MASK), txq->ntfy_addr); 231 } 232 233 num_queues = dev->data->nb_rx_queues; 234 priv->rxqs = (struct gve_rx_queue **)dev->data->rx_queues; 235 err = gve_adminq_create_rx_queues(priv, num_queues); 236 if (err) { 237 PMD_DRV_LOG(ERR, "failed to create %u rx queues.", num_queues); 238 goto err_tx; 239 } 240 for (i = 0; i < num_queues; i++) { 241 rxq = priv->rxqs[i]; 242 rxq->qrx_tail = 243 &priv->db_bar2[rte_be_to_cpu_32(rxq->qres->db_index)]; 244 245 rte_write32(rte_cpu_to_be_32(GVE_IRQ_MASK), rxq->ntfy_addr); 246 247 if (gve_is_gqi(priv)) 248 err = gve_refill_pages(rxq); 249 else 250 err = gve_refill_dqo(rxq); 251 if (err) { 252 PMD_DRV_LOG(ERR, "Failed to refill for RX"); 253 goto err_rx; 254 } 255 } 256 257 dev->data->dev_started = 1; 258 gve_link_update(dev, 0); 259 260 return 0; 261 262 err_rx: 263 gve_stop_rx_queues(dev); 264 err_tx: 265 gve_stop_tx_queues(dev); 266 return err; 267 } 268 269 static int 270 gve_dev_stop(struct rte_eth_dev *dev) 271 { 272 dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; 273 274 gve_stop_tx_queues(dev); 275 gve_stop_rx_queues(dev); 276 277 dev->data->dev_started = 0; 278 279 return 0; 280 } 281 282 static int 283 gve_dev_close(struct rte_eth_dev *dev) 284 { 285 struct gve_priv *priv = dev->data->dev_private; 286 int err = 0; 287 uint16_t i; 288 289 if (dev->data->dev_started) { 290 err = gve_dev_stop(dev); 291 if (err != 0) 292 PMD_DRV_LOG(ERR, "Failed to stop dev."); 293 } 294 295 if (gve_is_gqi(priv)) { 296 for (i = 0; i < dev->data->nb_tx_queues; i++) 297 gve_tx_queue_release(dev, i); 298 299 for (i = 0; i < dev->data->nb_rx_queues; i++) 300 gve_rx_queue_release(dev, i); 301 } else { 302 for (i = 0; i < dev->data->nb_tx_queues; i++) 303 gve_tx_queue_release_dqo(dev, i); 304 305 for (i = 0; i < dev->data->nb_rx_queues; i++) 306 gve_rx_queue_release_dqo(dev, i); 307 } 308 309 gve_free_qpls(priv); 310 rte_free(priv->adminq); 311 312 dev->data->mac_addrs = NULL; 313 314 return err; 315 } 316 317 static int 318 gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 319 { 320 struct gve_priv *priv = dev->data->dev_private; 321 322 dev_info->device = dev->device; 323 dev_info->max_mac_addrs = 1; 324 dev_info->max_rx_queues = priv->max_nb_rxq; 325 dev_info->max_tx_queues = priv->max_nb_txq; 326 dev_info->min_rx_bufsize = GVE_MIN_BUF_SIZE; 327 dev_info->max_rx_pktlen = GVE_MAX_RX_PKTLEN; 328 dev_info->max_mtu = GVE_MAX_MTU; 329 dev_info->min_mtu = GVE_MIN_MTU; 330 331 dev_info->rx_offload_capa = 0; 332 dev_info->tx_offload_capa = 333 RTE_ETH_TX_OFFLOAD_MULTI_SEGS | 334 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 335 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 336 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 337 RTE_ETH_TX_OFFLOAD_TCP_TSO; 338 339 if (priv->queue_format == GVE_DQO_RDA_FORMAT) 340 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO; 341 342 dev_info->default_rxconf = (struct rte_eth_rxconf) { 343 .rx_free_thresh = GVE_DEFAULT_RX_FREE_THRESH, 344 .rx_drop_en = 0, 345 .offloads = 0, 346 }; 347 348 dev_info->default_txconf = (struct rte_eth_txconf) { 349 .tx_free_thresh = GVE_DEFAULT_TX_FREE_THRESH, 350 .tx_rs_thresh = GVE_DEFAULT_TX_RS_THRESH, 351 .offloads = 0, 352 }; 353 354 dev_info->default_rxportconf.ring_size = priv->rx_desc_cnt; 355 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 356 .nb_max = priv->rx_desc_cnt, 357 .nb_min = priv->rx_desc_cnt, 358 .nb_align = 1, 359 }; 360 361 dev_info->default_txportconf.ring_size = priv->tx_desc_cnt; 362 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 363 .nb_max = priv->tx_desc_cnt, 364 .nb_min = priv->tx_desc_cnt, 365 .nb_align = 1, 366 }; 367 368 return 0; 369 } 370 371 static int 372 gve_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 373 { 374 uint16_t i; 375 376 for (i = 0; i < dev->data->nb_tx_queues; i++) { 377 struct gve_tx_queue *txq = dev->data->tx_queues[i]; 378 if (txq == NULL) 379 continue; 380 381 stats->opackets += txq->stats.packets; 382 stats->obytes += txq->stats.bytes; 383 stats->oerrors += txq->stats.errors; 384 } 385 386 for (i = 0; i < dev->data->nb_rx_queues; i++) { 387 struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 388 if (rxq == NULL) 389 continue; 390 391 stats->ipackets += rxq->stats.packets; 392 stats->ibytes += rxq->stats.bytes; 393 stats->ierrors += rxq->stats.errors; 394 stats->rx_nombuf += rxq->stats.no_mbufs; 395 } 396 397 return 0; 398 } 399 400 static int 401 gve_dev_stats_reset(struct rte_eth_dev *dev) 402 { 403 uint16_t i; 404 405 for (i = 0; i < dev->data->nb_tx_queues; i++) { 406 struct gve_tx_queue *txq = dev->data->tx_queues[i]; 407 if (txq == NULL) 408 continue; 409 410 memset(&txq->stats, 0, sizeof(txq->stats)); 411 } 412 413 for (i = 0; i < dev->data->nb_rx_queues; i++) { 414 struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 415 if (rxq == NULL) 416 continue; 417 418 memset(&rxq->stats, 0, sizeof(rxq->stats)); 419 } 420 421 return 0; 422 } 423 424 static int 425 gve_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 426 { 427 struct gve_priv *priv = dev->data->dev_private; 428 int err; 429 430 if (mtu < RTE_ETHER_MIN_MTU || mtu > priv->max_mtu) { 431 PMD_DRV_LOG(ERR, "MIN MTU is %u, MAX MTU is %u", 432 RTE_ETHER_MIN_MTU, priv->max_mtu); 433 return -EINVAL; 434 } 435 436 /* mtu setting is forbidden if port is start */ 437 if (dev->data->dev_started) { 438 PMD_DRV_LOG(ERR, "Port must be stopped before configuration"); 439 return -EBUSY; 440 } 441 442 err = gve_adminq_set_mtu(priv, mtu); 443 if (err) { 444 PMD_DRV_LOG(ERR, "Failed to set mtu as %u err = %d", mtu, err); 445 return err; 446 } 447 448 return 0; 449 } 450 451 #define TX_QUEUE_STATS_OFFSET(x) offsetof(struct gve_tx_stats, x) 452 #define RX_QUEUE_STATS_OFFSET(x) offsetof(struct gve_rx_stats, x) 453 454 static const struct gve_xstats_name_offset tx_xstats_name_offset[] = { 455 { "packets", TX_QUEUE_STATS_OFFSET(packets) }, 456 { "bytes", TX_QUEUE_STATS_OFFSET(bytes) }, 457 { "errors", TX_QUEUE_STATS_OFFSET(errors) }, 458 }; 459 460 static const struct gve_xstats_name_offset rx_xstats_name_offset[] = { 461 { "packets", RX_QUEUE_STATS_OFFSET(packets) }, 462 { "bytes", RX_QUEUE_STATS_OFFSET(bytes) }, 463 { "errors", RX_QUEUE_STATS_OFFSET(errors) }, 464 { "mbuf_alloc_errors", RX_QUEUE_STATS_OFFSET(no_mbufs) }, 465 { "mbuf_alloc_errors_bulk", RX_QUEUE_STATS_OFFSET(no_mbufs_bulk) }, 466 }; 467 468 static int 469 gve_xstats_count(struct rte_eth_dev *dev) 470 { 471 uint16_t i, count = 0; 472 473 for (i = 0; i < dev->data->nb_tx_queues; i++) { 474 if (dev->data->tx_queues[i]) 475 count += RTE_DIM(tx_xstats_name_offset); 476 } 477 478 for (i = 0; i < dev->data->nb_rx_queues; i++) { 479 if (dev->data->rx_queues[i]) 480 count += RTE_DIM(rx_xstats_name_offset); 481 } 482 483 return count; 484 } 485 486 static int 487 gve_xstats_get(struct rte_eth_dev *dev, 488 struct rte_eth_xstat *xstats, 489 unsigned int size) 490 { 491 uint16_t i, j, count = gve_xstats_count(dev); 492 const char *stats; 493 494 if (xstats == NULL || size < count) 495 return count; 496 497 count = 0; 498 499 for (i = 0; i < dev->data->nb_tx_queues; i++) { 500 const struct gve_tx_queue *txq = dev->data->tx_queues[i]; 501 if (txq == NULL) 502 continue; 503 504 stats = (const char *)&txq->stats; 505 for (j = 0; j < RTE_DIM(tx_xstats_name_offset); j++, count++) { 506 xstats[count].id = count; 507 xstats[count].value = *(const uint64_t *) 508 (stats + tx_xstats_name_offset[j].offset); 509 } 510 } 511 512 for (i = 0; i < dev->data->nb_rx_queues; i++) { 513 const struct gve_rx_queue *rxq = dev->data->rx_queues[i]; 514 if (rxq == NULL) 515 continue; 516 517 stats = (const char *)&rxq->stats; 518 for (j = 0; j < RTE_DIM(rx_xstats_name_offset); j++, count++) { 519 xstats[count].id = count; 520 xstats[count].value = *(const uint64_t *) 521 (stats + rx_xstats_name_offset[j].offset); 522 } 523 } 524 525 return count; 526 } 527 528 static int 529 gve_xstats_get_names(struct rte_eth_dev *dev, 530 struct rte_eth_xstat_name *xstats_names, 531 unsigned int size) 532 { 533 uint16_t i, j, count = gve_xstats_count(dev); 534 535 if (xstats_names == NULL || size < count) 536 return count; 537 538 count = 0; 539 540 for (i = 0; i < dev->data->nb_tx_queues; i++) { 541 if (dev->data->tx_queues[i] == NULL) 542 continue; 543 544 for (j = 0; j < RTE_DIM(tx_xstats_name_offset); j++) 545 snprintf(xstats_names[count++].name, 546 RTE_ETH_XSTATS_NAME_SIZE, 547 "tx_q%u_%s", i, tx_xstats_name_offset[j].name); 548 } 549 550 for (i = 0; i < dev->data->nb_rx_queues; i++) { 551 if (dev->data->rx_queues[i] == NULL) 552 continue; 553 554 for (j = 0; j < RTE_DIM(rx_xstats_name_offset); j++) 555 snprintf(xstats_names[count++].name, 556 RTE_ETH_XSTATS_NAME_SIZE, 557 "rx_q%u_%s", i, rx_xstats_name_offset[j].name); 558 } 559 560 return count; 561 } 562 563 static const struct eth_dev_ops gve_eth_dev_ops = { 564 .dev_configure = gve_dev_configure, 565 .dev_start = gve_dev_start, 566 .dev_stop = gve_dev_stop, 567 .dev_close = gve_dev_close, 568 .dev_infos_get = gve_dev_info_get, 569 .rx_queue_setup = gve_rx_queue_setup, 570 .tx_queue_setup = gve_tx_queue_setup, 571 .rx_queue_release = gve_rx_queue_release, 572 .tx_queue_release = gve_tx_queue_release, 573 .link_update = gve_link_update, 574 .stats_get = gve_dev_stats_get, 575 .stats_reset = gve_dev_stats_reset, 576 .mtu_set = gve_dev_mtu_set, 577 .xstats_get = gve_xstats_get, 578 .xstats_get_names = gve_xstats_get_names, 579 }; 580 581 static const struct eth_dev_ops gve_eth_dev_ops_dqo = { 582 .dev_configure = gve_dev_configure, 583 .dev_start = gve_dev_start, 584 .dev_stop = gve_dev_stop, 585 .dev_close = gve_dev_close, 586 .dev_infos_get = gve_dev_info_get, 587 .rx_queue_setup = gve_rx_queue_setup_dqo, 588 .tx_queue_setup = gve_tx_queue_setup_dqo, 589 .rx_queue_release = gve_rx_queue_release_dqo, 590 .tx_queue_release = gve_tx_queue_release_dqo, 591 .link_update = gve_link_update, 592 .stats_get = gve_dev_stats_get, 593 .stats_reset = gve_dev_stats_reset, 594 .mtu_set = gve_dev_mtu_set, 595 .xstats_get = gve_xstats_get, 596 .xstats_get_names = gve_xstats_get_names, 597 }; 598 599 static void 600 gve_free_counter_array(struct gve_priv *priv) 601 { 602 rte_memzone_free(priv->cnt_array_mz); 603 priv->cnt_array = NULL; 604 } 605 606 static void 607 gve_free_irq_db(struct gve_priv *priv) 608 { 609 rte_memzone_free(priv->irq_dbs_mz); 610 priv->irq_dbs = NULL; 611 } 612 613 static void 614 gve_teardown_device_resources(struct gve_priv *priv) 615 { 616 int err; 617 618 /* Tell device its resources are being freed */ 619 if (gve_get_device_resources_ok(priv)) { 620 err = gve_adminq_deconfigure_device_resources(priv); 621 if (err) 622 PMD_DRV_LOG(ERR, "Could not deconfigure device resources: err=%d", err); 623 } 624 gve_free_counter_array(priv); 625 gve_free_irq_db(priv); 626 gve_clear_device_resources_ok(priv); 627 } 628 629 static uint8_t 630 pci_dev_find_capability(struct rte_pci_device *pdev, int cap) 631 { 632 uint8_t pos, id; 633 uint16_t ent; 634 int loops; 635 int ret; 636 637 ret = rte_pci_read_config(pdev, &pos, sizeof(pos), PCI_CAPABILITY_LIST); 638 if (ret != sizeof(pos)) 639 return 0; 640 641 loops = (PCI_CFG_SPACE_SIZE - PCI_STD_HEADER_SIZEOF) / PCI_CAP_SIZEOF; 642 643 while (pos && loops--) { 644 ret = rte_pci_read_config(pdev, &ent, sizeof(ent), pos); 645 if (ret != sizeof(ent)) 646 return 0; 647 648 id = ent & 0xff; 649 if (id == 0xff) 650 break; 651 652 if (id == cap) 653 return pos; 654 655 pos = (ent >> 8); 656 } 657 658 return 0; 659 } 660 661 static int 662 pci_dev_msix_vec_count(struct rte_pci_device *pdev) 663 { 664 uint8_t msix_cap = pci_dev_find_capability(pdev, PCI_CAP_ID_MSIX); 665 uint16_t control; 666 int ret; 667 668 if (!msix_cap) 669 return 0; 670 671 ret = rte_pci_read_config(pdev, &control, sizeof(control), msix_cap + PCI_MSIX_FLAGS); 672 if (ret != sizeof(control)) 673 return 0; 674 675 return (control & PCI_MSIX_FLAGS_QSIZE) + 1; 676 } 677 678 static int 679 gve_setup_device_resources(struct gve_priv *priv) 680 { 681 char z_name[RTE_MEMZONE_NAMESIZE]; 682 const struct rte_memzone *mz; 683 int err = 0; 684 685 snprintf(z_name, sizeof(z_name), "gve_%s_cnt_arr", priv->pci_dev->device.name); 686 mz = rte_memzone_reserve_aligned(z_name, 687 priv->num_event_counters * sizeof(*priv->cnt_array), 688 rte_socket_id(), RTE_MEMZONE_IOVA_CONTIG, 689 PAGE_SIZE); 690 if (mz == NULL) { 691 PMD_DRV_LOG(ERR, "Could not alloc memzone for count array"); 692 return -ENOMEM; 693 } 694 priv->cnt_array = (rte_be32_t *)mz->addr; 695 priv->cnt_array_mz = mz; 696 697 snprintf(z_name, sizeof(z_name), "gve_%s_irqmz", priv->pci_dev->device.name); 698 mz = rte_memzone_reserve_aligned(z_name, 699 sizeof(*priv->irq_dbs) * (priv->num_ntfy_blks), 700 rte_socket_id(), RTE_MEMZONE_IOVA_CONTIG, 701 PAGE_SIZE); 702 if (mz == NULL) { 703 PMD_DRV_LOG(ERR, "Could not alloc memzone for irq_dbs"); 704 err = -ENOMEM; 705 goto free_cnt_array; 706 } 707 priv->irq_dbs = (struct gve_irq_db *)mz->addr; 708 priv->irq_dbs_mz = mz; 709 710 err = gve_adminq_configure_device_resources(priv, 711 priv->cnt_array_mz->iova, 712 priv->num_event_counters, 713 priv->irq_dbs_mz->iova, 714 priv->num_ntfy_blks); 715 if (unlikely(err)) { 716 PMD_DRV_LOG(ERR, "Could not config device resources: err=%d", err); 717 goto free_irq_dbs; 718 } 719 return 0; 720 721 free_irq_dbs: 722 gve_free_irq_db(priv); 723 free_cnt_array: 724 gve_free_counter_array(priv); 725 726 return err; 727 } 728 729 static int 730 gve_init_priv(struct gve_priv *priv, bool skip_describe_device) 731 { 732 uint16_t pages; 733 int num_ntfy; 734 uint32_t i; 735 int err; 736 737 /* Set up the adminq */ 738 err = gve_adminq_alloc(priv); 739 if (err) { 740 PMD_DRV_LOG(ERR, "Failed to alloc admin queue: err=%d", err); 741 return err; 742 } 743 744 if (skip_describe_device) 745 goto setup_device; 746 747 /* Get the initial information we need from the device */ 748 err = gve_adminq_describe_device(priv); 749 if (err) { 750 PMD_DRV_LOG(ERR, "Could not get device information: err=%d", err); 751 goto free_adminq; 752 } 753 754 num_ntfy = pci_dev_msix_vec_count(priv->pci_dev); 755 if (num_ntfy <= 0) { 756 PMD_DRV_LOG(ERR, "Could not count MSI-x vectors"); 757 err = -EIO; 758 goto free_adminq; 759 } else if (num_ntfy < GVE_MIN_MSIX) { 760 PMD_DRV_LOG(ERR, "GVE needs at least %d MSI-x vectors, but only has %d", 761 GVE_MIN_MSIX, num_ntfy); 762 err = -EINVAL; 763 goto free_adminq; 764 } 765 766 priv->num_registered_pages = 0; 767 768 /* gvnic has one Notification Block per MSI-x vector, except for the 769 * management vector 770 */ 771 priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1; 772 priv->mgmt_msix_idx = priv->num_ntfy_blks; 773 774 priv->max_nb_txq = RTE_MIN(priv->max_nb_txq, priv->num_ntfy_blks / 2); 775 priv->max_nb_rxq = RTE_MIN(priv->max_nb_rxq, priv->num_ntfy_blks / 2); 776 777 if (priv->default_num_queues > 0) { 778 priv->max_nb_txq = RTE_MIN(priv->default_num_queues, priv->max_nb_txq); 779 priv->max_nb_rxq = RTE_MIN(priv->default_num_queues, priv->max_nb_rxq); 780 } 781 782 PMD_DRV_LOG(INFO, "Max TX queues %d, Max RX queues %d", 783 priv->max_nb_txq, priv->max_nb_rxq); 784 785 /* In GQI_QPL queue format: 786 * Allocate queue page lists according to max queue number 787 * tx qpl id should start from 0 while rx qpl id should start 788 * from priv->max_nb_txq 789 */ 790 if (priv->queue_format == GVE_GQI_QPL_FORMAT) { 791 priv->qpl = rte_zmalloc("gve_qpl", 792 (priv->max_nb_txq + priv->max_nb_rxq) * 793 sizeof(struct gve_queue_page_list), 0); 794 if (priv->qpl == NULL) { 795 PMD_DRV_LOG(ERR, "Failed to alloc qpl."); 796 err = -ENOMEM; 797 goto free_adminq; 798 } 799 800 for (i = 0; i < priv->max_nb_txq + priv->max_nb_rxq; i++) { 801 if (i < priv->max_nb_txq) 802 pages = priv->tx_pages_per_qpl; 803 else 804 pages = priv->rx_data_slot_cnt; 805 err = gve_alloc_queue_page_list(priv, i, pages); 806 if (err != 0) { 807 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u.", i); 808 goto err_qpl; 809 } 810 } 811 } 812 813 setup_device: 814 err = gve_setup_device_resources(priv); 815 if (!err) 816 return 0; 817 err_qpl: 818 gve_free_qpls(priv); 819 free_adminq: 820 gve_adminq_free(priv); 821 return err; 822 } 823 824 static void 825 gve_teardown_priv_resources(struct gve_priv *priv) 826 { 827 gve_teardown_device_resources(priv); 828 gve_adminq_free(priv); 829 } 830 831 static int 832 gve_dev_init(struct rte_eth_dev *eth_dev) 833 { 834 struct gve_priv *priv = eth_dev->data->dev_private; 835 int max_tx_queues, max_rx_queues; 836 struct rte_pci_device *pci_dev; 837 struct gve_registers *reg_bar; 838 rte_be32_t *db_bar; 839 int err; 840 841 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 842 return 0; 843 844 pci_dev = RTE_DEV_TO_PCI(eth_dev->device); 845 846 reg_bar = pci_dev->mem_resource[GVE_REG_BAR].addr; 847 if (!reg_bar) { 848 PMD_DRV_LOG(ERR, "Failed to map pci bar!"); 849 return -ENOMEM; 850 } 851 852 db_bar = pci_dev->mem_resource[GVE_DB_BAR].addr; 853 if (!db_bar) { 854 PMD_DRV_LOG(ERR, "Failed to map doorbell bar!"); 855 return -ENOMEM; 856 } 857 858 gve_write_version(®_bar->driver_version); 859 /* Get max queues to alloc etherdev */ 860 max_tx_queues = ioread32be(®_bar->max_tx_queues); 861 max_rx_queues = ioread32be(®_bar->max_rx_queues); 862 863 priv->reg_bar0 = reg_bar; 864 priv->db_bar2 = db_bar; 865 priv->pci_dev = pci_dev; 866 priv->state_flags = 0x0; 867 868 priv->max_nb_txq = max_tx_queues; 869 priv->max_nb_rxq = max_rx_queues; 870 871 err = gve_init_priv(priv, false); 872 if (err) 873 return err; 874 875 if (gve_is_gqi(priv)) { 876 eth_dev->dev_ops = &gve_eth_dev_ops; 877 eth_dev->rx_pkt_burst = gve_rx_burst; 878 eth_dev->tx_pkt_burst = gve_tx_burst; 879 } else { 880 eth_dev->dev_ops = &gve_eth_dev_ops_dqo; 881 eth_dev->tx_pkt_burst = gve_tx_burst_dqo; 882 } 883 884 eth_dev->data->mac_addrs = &priv->dev_addr; 885 886 return 0; 887 } 888 889 static int 890 gve_dev_uninit(struct rte_eth_dev *eth_dev) 891 { 892 struct gve_priv *priv = eth_dev->data->dev_private; 893 894 gve_teardown_priv_resources(priv); 895 896 eth_dev->data->mac_addrs = NULL; 897 898 return 0; 899 } 900 901 static int 902 gve_pci_probe(__rte_unused struct rte_pci_driver *pci_drv, 903 struct rte_pci_device *pci_dev) 904 { 905 return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct gve_priv), gve_dev_init); 906 } 907 908 static int 909 gve_pci_remove(struct rte_pci_device *pci_dev) 910 { 911 return rte_eth_dev_pci_generic_remove(pci_dev, gve_dev_uninit); 912 } 913 914 static const struct rte_pci_id pci_id_gve_map[] = { 915 { RTE_PCI_DEVICE(GOOGLE_VENDOR_ID, GVE_DEV_ID) }, 916 { .device_id = 0 }, 917 }; 918 919 static struct rte_pci_driver rte_gve_pmd = { 920 .id_table = pci_id_gve_map, 921 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 922 .probe = gve_pci_probe, 923 .remove = gve_pci_remove, 924 }; 925 926 RTE_PMD_REGISTER_PCI(net_gve, rte_gve_pmd); 927 RTE_PMD_REGISTER_PCI_TABLE(net_gve, pci_id_gve_map); 928 RTE_PMD_REGISTER_KMOD_DEP(net_gve, "* igb_uio | vfio-pci"); 929 RTE_LOG_REGISTER_SUFFIX(gve_logtype_driver, driver, NOTICE); 930