1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/queue.h> 35 #include <stdio.h> 36 #include <errno.h> 37 #include <stdint.h> 38 #include <string.h> 39 #include <unistd.h> 40 #include <stdarg.h> 41 #include <fcntl.h> 42 #include <inttypes.h> 43 #include <rte_byteorder.h> 44 #include <rte_common.h> 45 #include <rte_cycles.h> 46 47 #include <rte_interrupts.h> 48 #include <rte_log.h> 49 #include <rte_debug.h> 50 #include <rte_pci.h> 51 #include <rte_bus_pci.h> 52 #include <rte_atomic.h> 53 #include <rte_branch_prediction.h> 54 #include <rte_memory.h> 55 #include <rte_memzone.h> 56 #include <rte_eal.h> 57 #include <rte_alarm.h> 58 #include <rte_ether.h> 59 #include <rte_ethdev.h> 60 #include <rte_ethdev_pci.h> 61 #include <rte_string_fns.h> 62 #include <rte_malloc.h> 63 #include <rte_dev.h> 64 65 #include "base/vmxnet3_defs.h" 66 67 #include "vmxnet3_ring.h" 68 #include "vmxnet3_logs.h" 69 #include "vmxnet3_ethdev.h" 70 71 #define PROCESS_SYS_EVENTS 0 72 73 #define VMXNET3_TX_MAX_SEG UINT8_MAX 74 75 static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev); 76 static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev); 77 static int vmxnet3_dev_configure(struct rte_eth_dev *dev); 78 static int vmxnet3_dev_start(struct rte_eth_dev *dev); 79 static void vmxnet3_dev_stop(struct rte_eth_dev *dev); 80 static void vmxnet3_dev_close(struct rte_eth_dev *dev); 81 static void vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set); 82 static void vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev); 83 static void vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev); 84 static void vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev); 85 static void vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev); 86 static int __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 87 int wait_to_complete); 88 static int vmxnet3_dev_link_update(struct rte_eth_dev *dev, 89 int wait_to_complete); 90 static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw); 91 static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev, 92 struct rte_eth_stats *stats); 93 static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 94 struct rte_eth_xstat_name *xstats, 95 unsigned int n); 96 static int vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, 97 struct rte_eth_xstat *xstats, unsigned int n); 98 static void vmxnet3_dev_info_get(struct rte_eth_dev *dev, 99 struct rte_eth_dev_info *dev_info); 100 static const uint32_t * 101 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev); 102 static int vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, 103 uint16_t vid, int on); 104 static int vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 105 static void vmxnet3_mac_addr_set(struct rte_eth_dev *dev, 106 struct ether_addr *mac_addr); 107 static void vmxnet3_interrupt_handler(void *param); 108 109 /* 110 * The set of PCI devices this driver supports 111 */ 112 #define VMWARE_PCI_VENDOR_ID 0x15AD 113 #define VMWARE_DEV_ID_VMXNET3 0x07B0 114 static const struct rte_pci_id pci_id_vmxnet3_map[] = { 115 { RTE_PCI_DEVICE(VMWARE_PCI_VENDOR_ID, VMWARE_DEV_ID_VMXNET3) }, 116 { .vendor_id = 0, /* sentinel */ }, 117 }; 118 119 static const struct eth_dev_ops vmxnet3_eth_dev_ops = { 120 .dev_configure = vmxnet3_dev_configure, 121 .dev_start = vmxnet3_dev_start, 122 .dev_stop = vmxnet3_dev_stop, 123 .dev_close = vmxnet3_dev_close, 124 .promiscuous_enable = vmxnet3_dev_promiscuous_enable, 125 .promiscuous_disable = vmxnet3_dev_promiscuous_disable, 126 .allmulticast_enable = vmxnet3_dev_allmulticast_enable, 127 .allmulticast_disable = vmxnet3_dev_allmulticast_disable, 128 .link_update = vmxnet3_dev_link_update, 129 .stats_get = vmxnet3_dev_stats_get, 130 .xstats_get_names = vmxnet3_dev_xstats_get_names, 131 .xstats_get = vmxnet3_dev_xstats_get, 132 .mac_addr_set = vmxnet3_mac_addr_set, 133 .dev_infos_get = vmxnet3_dev_info_get, 134 .dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get, 135 .vlan_filter_set = vmxnet3_dev_vlan_filter_set, 136 .vlan_offload_set = vmxnet3_dev_vlan_offload_set, 137 .rx_queue_setup = vmxnet3_dev_rx_queue_setup, 138 .rx_queue_release = vmxnet3_dev_rx_queue_release, 139 .tx_queue_setup = vmxnet3_dev_tx_queue_setup, 140 .tx_queue_release = vmxnet3_dev_tx_queue_release, 141 }; 142 143 struct vmxnet3_xstats_name_off { 144 char name[RTE_ETH_XSTATS_NAME_SIZE]; 145 unsigned int offset; 146 }; 147 148 /* tx_qX_ is prepended to the name string here */ 149 static const struct vmxnet3_xstats_name_off vmxnet3_txq_stat_strings[] = { 150 {"drop_total", offsetof(struct vmxnet3_txq_stats, drop_total)}, 151 {"drop_too_many_segs", offsetof(struct vmxnet3_txq_stats, drop_too_many_segs)}, 152 {"drop_tso", offsetof(struct vmxnet3_txq_stats, drop_tso)}, 153 {"tx_ring_full", offsetof(struct vmxnet3_txq_stats, tx_ring_full)}, 154 }; 155 156 /* rx_qX_ is prepended to the name string here */ 157 static const struct vmxnet3_xstats_name_off vmxnet3_rxq_stat_strings[] = { 158 {"drop_total", offsetof(struct vmxnet3_rxq_stats, drop_total)}, 159 {"drop_err", offsetof(struct vmxnet3_rxq_stats, drop_err)}, 160 {"drop_fcs", offsetof(struct vmxnet3_rxq_stats, drop_fcs)}, 161 {"rx_buf_alloc_failure", offsetof(struct vmxnet3_rxq_stats, rx_buf_alloc_failure)}, 162 }; 163 164 static const struct rte_memzone * 165 gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size, 166 const char *post_string, int socket_id, 167 uint16_t align, bool reuse) 168 { 169 char z_name[RTE_MEMZONE_NAMESIZE]; 170 const struct rte_memzone *mz; 171 172 snprintf(z_name, sizeof(z_name), "%s_%d_%s", 173 dev->device->driver->name, dev->data->port_id, post_string); 174 175 mz = rte_memzone_lookup(z_name); 176 if (!reuse) { 177 if (mz) 178 rte_memzone_free(mz); 179 return rte_memzone_reserve_aligned(z_name, size, socket_id, 180 0, align); 181 } 182 183 if (mz) 184 return mz; 185 186 return rte_memzone_reserve_aligned(z_name, size, socket_id, 0, align); 187 } 188 189 /** 190 * Atomically reads the link status information from global 191 * structure rte_eth_dev. 192 * 193 * @param dev 194 * - Pointer to the structure rte_eth_dev to read from. 195 * - Pointer to the buffer to be saved with the link status. 196 * 197 * @return 198 * - On success, zero. 199 * - On failure, negative value. 200 */ 201 202 static int 203 vmxnet3_dev_atomic_read_link_status(struct rte_eth_dev *dev, 204 struct rte_eth_link *link) 205 { 206 struct rte_eth_link *dst = link; 207 struct rte_eth_link *src = &(dev->data->dev_link); 208 209 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 210 *(uint64_t *)src) == 0) 211 return -1; 212 213 return 0; 214 } 215 216 /** 217 * Atomically writes the link status information into global 218 * structure rte_eth_dev. 219 * 220 * @param dev 221 * - Pointer to the structure rte_eth_dev to write to. 222 * - Pointer to the buffer to be saved with the link status. 223 * 224 * @return 225 * - On success, zero. 226 * - On failure, negative value. 227 */ 228 static int 229 vmxnet3_dev_atomic_write_link_status(struct rte_eth_dev *dev, 230 struct rte_eth_link *link) 231 { 232 struct rte_eth_link *dst = &(dev->data->dev_link); 233 struct rte_eth_link *src = link; 234 235 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 236 *(uint64_t *)src) == 0) 237 return -1; 238 239 return 0; 240 } 241 242 /* 243 * This function is based on vmxnet3_disable_intr() 244 */ 245 static void 246 vmxnet3_disable_intr(struct vmxnet3_hw *hw) 247 { 248 int i; 249 250 PMD_INIT_FUNC_TRACE(); 251 252 hw->shared->devRead.intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL; 253 for (i = 0; i < hw->num_intrs; i++) 254 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 1); 255 } 256 257 static void 258 vmxnet3_enable_intr(struct vmxnet3_hw *hw) 259 { 260 int i; 261 262 PMD_INIT_FUNC_TRACE(); 263 264 hw->shared->devRead.intrConf.intrCtrl &= ~VMXNET3_IC_DISABLE_ALL; 265 for (i = 0; i < hw->num_intrs; i++) 266 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + i * 8, 0); 267 } 268 269 /* 270 * Gets tx data ring descriptor size. 271 */ 272 static uint16_t 273 eth_vmxnet3_txdata_get(struct vmxnet3_hw *hw) 274 { 275 uint16 txdata_desc_size; 276 277 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 278 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 279 txdata_desc_size = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 280 281 return (txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE || 282 txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE || 283 txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK) ? 284 sizeof(struct Vmxnet3_TxDataDesc) : txdata_desc_size; 285 } 286 287 /* 288 * It returns 0 on success. 289 */ 290 static int 291 eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) 292 { 293 struct rte_pci_device *pci_dev; 294 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 295 uint32_t mac_hi, mac_lo, ver; 296 297 PMD_INIT_FUNC_TRACE(); 298 299 eth_dev->dev_ops = &vmxnet3_eth_dev_ops; 300 eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts; 301 eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts; 302 eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts; 303 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 304 305 /* 306 * for secondary processes, we don't initialize any further as primary 307 * has already done this work. 308 */ 309 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 310 return 0; 311 312 rte_eth_copy_pci_info(eth_dev, pci_dev); 313 314 /* Vendor and Device ID need to be set before init of shared code */ 315 hw->device_id = pci_dev->id.device_id; 316 hw->vendor_id = pci_dev->id.vendor_id; 317 hw->hw_addr0 = (void *)pci_dev->mem_resource[0].addr; 318 hw->hw_addr1 = (void *)pci_dev->mem_resource[1].addr; 319 320 hw->num_rx_queues = 1; 321 hw->num_tx_queues = 1; 322 hw->bufs_per_pkt = 1; 323 324 /* Check h/w version compatibility with driver. */ 325 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_VRRS); 326 PMD_INIT_LOG(DEBUG, "Hardware version : %d", ver); 327 328 if (ver & (1 << VMXNET3_REV_3)) { 329 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 330 1 << VMXNET3_REV_3); 331 hw->version = VMXNET3_REV_3 + 1; 332 } else if (ver & (1 << VMXNET3_REV_2)) { 333 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 334 1 << VMXNET3_REV_2); 335 hw->version = VMXNET3_REV_2 + 1; 336 } else if (ver & (1 << VMXNET3_REV_1)) { 337 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 338 1 << VMXNET3_REV_1); 339 hw->version = VMXNET3_REV_1 + 1; 340 } else { 341 PMD_INIT_LOG(ERR, "Incompatible hardware version: %d", ver); 342 return -EIO; 343 } 344 345 PMD_INIT_LOG(DEBUG, "Using device version %d\n", hw->version); 346 347 /* Check UPT version compatibility with driver. */ 348 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_UVRS); 349 PMD_INIT_LOG(DEBUG, "UPT hardware version : %d", ver); 350 if (ver & 0x1) 351 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_UVRS, 1); 352 else { 353 PMD_INIT_LOG(ERR, "Incompatible UPT version."); 354 return -EIO; 355 } 356 357 /* Getting MAC Address */ 358 mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL); 359 mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH); 360 memcpy(hw->perm_addr, &mac_lo, 4); 361 memcpy(hw->perm_addr + 4, &mac_hi, 2); 362 363 /* Allocate memory for storing MAC addresses */ 364 eth_dev->data->mac_addrs = rte_zmalloc("vmxnet3", ETHER_ADDR_LEN * 365 VMXNET3_MAX_MAC_ADDRS, 0); 366 if (eth_dev->data->mac_addrs == NULL) { 367 PMD_INIT_LOG(ERR, 368 "Failed to allocate %d bytes needed to store MAC addresses", 369 ETHER_ADDR_LEN * VMXNET3_MAX_MAC_ADDRS); 370 return -ENOMEM; 371 } 372 /* Copy the permanent MAC address */ 373 ether_addr_copy((struct ether_addr *) hw->perm_addr, 374 ð_dev->data->mac_addrs[0]); 375 376 PMD_INIT_LOG(DEBUG, "MAC Address : %02x:%02x:%02x:%02x:%02x:%02x", 377 hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2], 378 hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]); 379 380 /* Put device in Quiesce Mode */ 381 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 382 383 /* allow untagged pkts */ 384 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, 0); 385 386 hw->txdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 387 eth_vmxnet3_txdata_get(hw) : sizeof(struct Vmxnet3_TxDataDesc); 388 389 hw->rxdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 390 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 391 RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) == 392 hw->rxdata_desc_size); 393 394 /* clear shadow stats */ 395 memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats)); 396 memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats)); 397 398 return 0; 399 } 400 401 static int 402 eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev) 403 { 404 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 405 406 PMD_INIT_FUNC_TRACE(); 407 408 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 409 return 0; 410 411 if (hw->adapter_stopped == 0) 412 vmxnet3_dev_close(eth_dev); 413 414 eth_dev->dev_ops = NULL; 415 eth_dev->rx_pkt_burst = NULL; 416 eth_dev->tx_pkt_burst = NULL; 417 eth_dev->tx_pkt_prepare = NULL; 418 419 rte_free(eth_dev->data->mac_addrs); 420 eth_dev->data->mac_addrs = NULL; 421 422 return 0; 423 } 424 425 static int eth_vmxnet3_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 426 struct rte_pci_device *pci_dev) 427 { 428 return rte_eth_dev_pci_generic_probe(pci_dev, 429 sizeof(struct vmxnet3_hw), eth_vmxnet3_dev_init); 430 } 431 432 static int eth_vmxnet3_pci_remove(struct rte_pci_device *pci_dev) 433 { 434 return rte_eth_dev_pci_generic_remove(pci_dev, eth_vmxnet3_dev_uninit); 435 } 436 437 static struct rte_pci_driver rte_vmxnet3_pmd = { 438 .id_table = pci_id_vmxnet3_map, 439 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 440 .probe = eth_vmxnet3_pci_probe, 441 .remove = eth_vmxnet3_pci_remove, 442 }; 443 444 static int 445 vmxnet3_dev_configure(struct rte_eth_dev *dev) 446 { 447 const struct rte_memzone *mz; 448 struct vmxnet3_hw *hw = dev->data->dev_private; 449 size_t size; 450 451 PMD_INIT_FUNC_TRACE(); 452 453 if (dev->data->nb_tx_queues > VMXNET3_MAX_TX_QUEUES || 454 dev->data->nb_rx_queues > VMXNET3_MAX_RX_QUEUES) { 455 PMD_INIT_LOG(ERR, "ERROR: Number of queues not supported"); 456 return -EINVAL; 457 } 458 459 if (!rte_is_power_of_2(dev->data->nb_rx_queues)) { 460 PMD_INIT_LOG(ERR, "ERROR: Number of rx queues not power of 2"); 461 return -EINVAL; 462 } 463 464 size = dev->data->nb_rx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 465 dev->data->nb_tx_queues * sizeof(struct Vmxnet3_RxQueueDesc); 466 467 if (size > UINT16_MAX) 468 return -EINVAL; 469 470 hw->num_rx_queues = (uint8_t)dev->data->nb_rx_queues; 471 hw->num_tx_queues = (uint8_t)dev->data->nb_tx_queues; 472 473 /* 474 * Allocate a memzone for Vmxnet3_DriverShared - Vmxnet3_DSDevRead 475 * on current socket 476 */ 477 mz = gpa_zone_reserve(dev, sizeof(struct Vmxnet3_DriverShared), 478 "shared", rte_socket_id(), 8, 1); 479 480 if (mz == NULL) { 481 PMD_INIT_LOG(ERR, "ERROR: Creating shared zone"); 482 return -ENOMEM; 483 } 484 memset(mz->addr, 0, mz->len); 485 486 hw->shared = mz->addr; 487 hw->sharedPA = mz->iova; 488 489 /* 490 * Allocate a memzone for Vmxnet3_RxQueueDesc - Vmxnet3_TxQueueDesc 491 * on current socket. 492 * 493 * We cannot reuse this memzone from previous allocation as its size 494 * depends on the number of tx and rx queues, which could be different 495 * from one config to another. 496 */ 497 mz = gpa_zone_reserve(dev, size, "queuedesc", rte_socket_id(), 498 VMXNET3_QUEUE_DESC_ALIGN, 0); 499 if (mz == NULL) { 500 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); 501 return -ENOMEM; 502 } 503 memset(mz->addr, 0, mz->len); 504 505 hw->tqd_start = (Vmxnet3_TxQueueDesc *)mz->addr; 506 hw->rqd_start = (Vmxnet3_RxQueueDesc *)(hw->tqd_start + hw->num_tx_queues); 507 508 hw->queueDescPA = mz->iova; 509 hw->queue_desc_len = (uint16_t)size; 510 511 if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) { 512 /* Allocate memory structure for UPT1_RSSConf and configure */ 513 mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf), 514 "rss_conf", rte_socket_id(), 515 RTE_CACHE_LINE_SIZE, 1); 516 if (mz == NULL) { 517 PMD_INIT_LOG(ERR, 518 "ERROR: Creating rss_conf structure zone"); 519 return -ENOMEM; 520 } 521 memset(mz->addr, 0, mz->len); 522 523 hw->rss_conf = mz->addr; 524 hw->rss_confPA = mz->iova; 525 } 526 527 return 0; 528 } 529 530 static void 531 vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr) 532 { 533 uint32_t val; 534 535 PMD_INIT_LOG(DEBUG, 536 "Writing MAC Address : %02x:%02x:%02x:%02x:%02x:%02x", 537 addr[0], addr[1], addr[2], 538 addr[3], addr[4], addr[5]); 539 540 memcpy(&val, addr, 4); 541 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACL, val); 542 543 memcpy(&val, addr + 4, 2); 544 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val); 545 } 546 547 static int 548 vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev) 549 { 550 struct vmxnet3_hw *hw = dev->data->dev_private; 551 Vmxnet3_DriverShared *shared = hw->shared; 552 Vmxnet3_CmdInfo *cmdInfo; 553 struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES]; 554 uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES]; 555 uint32_t num, i, j, size; 556 557 if (hw->memRegsPA == 0) { 558 const struct rte_memzone *mz; 559 560 size = sizeof(Vmxnet3_MemRegs) + 561 (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) * 562 sizeof(Vmxnet3_MemoryRegion); 563 564 mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8, 565 1); 566 if (mz == NULL) { 567 PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone"); 568 return -ENOMEM; 569 } 570 memset(mz->addr, 0, mz->len); 571 hw->memRegs = mz->addr; 572 hw->memRegsPA = mz->iova; 573 } 574 575 num = hw->num_rx_queues; 576 577 for (i = 0; i < num; i++) { 578 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 579 580 mp[i] = rxq->mp; 581 index[i] = 1 << i; 582 } 583 584 /* 585 * The same mempool could be used by multiple queues. In such a case, 586 * remove duplicate mempool entries. Only one entry is kept with 587 * bitmask indicating queues that are using this mempool. 588 */ 589 for (i = 1; i < num; i++) { 590 for (j = 0; j < i; j++) { 591 if (mp[i] == mp[j]) { 592 mp[i] = NULL; 593 index[j] |= 1 << i; 594 break; 595 } 596 } 597 } 598 599 j = 0; 600 for (i = 0; i < num; i++) { 601 if (mp[i] == NULL) 602 continue; 603 604 Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j]; 605 606 mr->startPA = 607 (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->iova; 608 mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ? 609 STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX; 610 mr->txQueueBits = index[i]; 611 mr->rxQueueBits = index[i]; 612 613 PMD_INIT_LOG(INFO, 614 "index: %u startPA: %" PRIu64 " length: %u, " 615 "rxBits: %x", 616 j, mr->startPA, mr->length, mr->rxQueueBits); 617 j++; 618 } 619 hw->memRegs->numRegs = j; 620 PMD_INIT_LOG(INFO, "numRegs: %u", j); 621 622 size = sizeof(Vmxnet3_MemRegs) + 623 (j - 1) * sizeof(Vmxnet3_MemoryRegion); 624 625 cmdInfo = &shared->cu.cmdInfo; 626 cmdInfo->varConf.confVer = 1; 627 cmdInfo->varConf.confLen = size; 628 cmdInfo->varConf.confPA = hw->memRegsPA; 629 630 return 0; 631 } 632 633 static int 634 vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) 635 { 636 struct rte_eth_conf port_conf = dev->data->dev_conf; 637 struct vmxnet3_hw *hw = dev->data->dev_private; 638 uint32_t mtu = dev->data->mtu; 639 Vmxnet3_DriverShared *shared = hw->shared; 640 Vmxnet3_DSDevRead *devRead = &shared->devRead; 641 uint32_t i; 642 int ret; 643 644 shared->magic = VMXNET3_REV1_MAGIC; 645 devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM; 646 647 /* Setting up Guest OS information */ 648 devRead->misc.driverInfo.gos.gosBits = sizeof(void *) == 4 ? 649 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64; 650 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 651 devRead->misc.driverInfo.vmxnet3RevSpt = 1; 652 devRead->misc.driverInfo.uptVerSpt = 1; 653 654 devRead->misc.mtu = rte_le_to_cpu_32(mtu); 655 devRead->misc.queueDescPA = hw->queueDescPA; 656 devRead->misc.queueDescLen = hw->queue_desc_len; 657 devRead->misc.numTxQueues = hw->num_tx_queues; 658 devRead->misc.numRxQueues = hw->num_rx_queues; 659 660 /* 661 * Set number of interrupts to 1 662 * PMD by default disables all the interrupts but this is MUST 663 * to activate device. It needs at least one interrupt for 664 * link events to handle 665 */ 666 hw->num_intrs = devRead->intrConf.numIntrs = 1; 667 devRead->intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL; 668 669 for (i = 0; i < hw->num_tx_queues; i++) { 670 Vmxnet3_TxQueueDesc *tqd = &hw->tqd_start[i]; 671 vmxnet3_tx_queue_t *txq = dev->data->tx_queues[i]; 672 673 tqd->ctrl.txNumDeferred = 0; 674 tqd->ctrl.txThreshold = 1; 675 tqd->conf.txRingBasePA = txq->cmd_ring.basePA; 676 tqd->conf.compRingBasePA = txq->comp_ring.basePA; 677 tqd->conf.dataRingBasePA = txq->data_ring.basePA; 678 679 tqd->conf.txRingSize = txq->cmd_ring.size; 680 tqd->conf.compRingSize = txq->comp_ring.size; 681 tqd->conf.dataRingSize = txq->data_ring.size; 682 tqd->conf.txDataRingDescSize = txq->txdata_desc_size; 683 tqd->conf.intrIdx = txq->comp_ring.intr_idx; 684 tqd->status.stopped = TRUE; 685 tqd->status.error = 0; 686 memset(&tqd->stats, 0, sizeof(tqd->stats)); 687 } 688 689 for (i = 0; i < hw->num_rx_queues; i++) { 690 Vmxnet3_RxQueueDesc *rqd = &hw->rqd_start[i]; 691 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 692 693 rqd->conf.rxRingBasePA[0] = rxq->cmd_ring[0].basePA; 694 rqd->conf.rxRingBasePA[1] = rxq->cmd_ring[1].basePA; 695 rqd->conf.compRingBasePA = rxq->comp_ring.basePA; 696 697 rqd->conf.rxRingSize[0] = rxq->cmd_ring[0].size; 698 rqd->conf.rxRingSize[1] = rxq->cmd_ring[1].size; 699 rqd->conf.compRingSize = rxq->comp_ring.size; 700 rqd->conf.intrIdx = rxq->comp_ring.intr_idx; 701 if (VMXNET3_VERSION_GE_3(hw)) { 702 rqd->conf.rxDataRingBasePA = rxq->data_ring.basePA; 703 rqd->conf.rxDataRingDescSize = rxq->data_desc_size; 704 } 705 rqd->status.stopped = TRUE; 706 rqd->status.error = 0; 707 memset(&rqd->stats, 0, sizeof(rqd->stats)); 708 } 709 710 /* RxMode set to 0 of VMXNET3_RXM_xxx */ 711 devRead->rxFilterConf.rxMode = 0; 712 713 /* Setting up feature flags */ 714 if (dev->data->dev_conf.rxmode.hw_ip_checksum) 715 devRead->misc.uptFeatures |= VMXNET3_F_RXCSUM; 716 717 if (dev->data->dev_conf.rxmode.enable_lro) { 718 devRead->misc.uptFeatures |= VMXNET3_F_LRO; 719 devRead->misc.maxNumRxSG = 0; 720 } 721 722 if (port_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) { 723 ret = vmxnet3_rss_configure(dev); 724 if (ret != VMXNET3_SUCCESS) 725 return ret; 726 727 devRead->misc.uptFeatures |= VMXNET3_F_RSS; 728 devRead->rssConfDesc.confVer = 1; 729 devRead->rssConfDesc.confLen = sizeof(struct VMXNET3_RSSConf); 730 devRead->rssConfDesc.confPA = hw->rss_confPA; 731 } 732 733 ret = vmxnet3_dev_vlan_offload_set(dev, 734 ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK); 735 if (ret) 736 return ret; 737 738 vmxnet3_write_mac(hw, dev->data->mac_addrs->addr_bytes); 739 740 return VMXNET3_SUCCESS; 741 } 742 743 /* 744 * Configure device link speed and setup link. 745 * Must be called after eth_vmxnet3_dev_init. Other wise it might fail 746 * It returns 0 on success. 747 */ 748 static int 749 vmxnet3_dev_start(struct rte_eth_dev *dev) 750 { 751 int ret; 752 struct vmxnet3_hw *hw = dev->data->dev_private; 753 754 PMD_INIT_FUNC_TRACE(); 755 756 /* Save stats before it is reset by CMD_ACTIVATE */ 757 vmxnet3_hw_stats_save(hw); 758 759 ret = vmxnet3_setup_driver_shared(dev); 760 if (ret != VMXNET3_SUCCESS) 761 return ret; 762 763 /* check if lsc interrupt feature is enabled */ 764 if (dev->data->dev_conf.intr_conf.lsc) { 765 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 766 767 /* Setup interrupt callback */ 768 rte_intr_callback_register(&pci_dev->intr_handle, 769 vmxnet3_interrupt_handler, dev); 770 771 if (rte_intr_enable(&pci_dev->intr_handle) < 0) { 772 PMD_INIT_LOG(ERR, "interrupt enable failed"); 773 return -EIO; 774 } 775 } 776 777 /* Exchange shared data with device */ 778 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 779 VMXNET3_GET_ADDR_LO(hw->sharedPA)); 780 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 781 VMXNET3_GET_ADDR_HI(hw->sharedPA)); 782 783 /* Activate device by register write */ 784 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV); 785 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 786 787 if (ret != 0) { 788 PMD_INIT_LOG(ERR, "Device activation: UNSUCCESSFUL"); 789 return -EINVAL; 790 } 791 792 /* Setup memory region for rx buffers */ 793 ret = vmxnet3_dev_setup_memreg(dev); 794 if (ret == 0) { 795 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 796 VMXNET3_CMD_REGISTER_MEMREGS); 797 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 798 if (ret != 0) 799 PMD_INIT_LOG(DEBUG, 800 "Failed in setup memory region cmd\n"); 801 ret = 0; 802 } else { 803 PMD_INIT_LOG(DEBUG, "Failed to setup memory region\n"); 804 } 805 806 /* Disable interrupts */ 807 vmxnet3_disable_intr(hw); 808 809 /* 810 * Load RX queues with blank mbufs and update next2fill index for device 811 * Update RxMode of the device 812 */ 813 ret = vmxnet3_dev_rxtx_init(dev); 814 if (ret != VMXNET3_SUCCESS) { 815 PMD_INIT_LOG(ERR, "Device queue init: UNSUCCESSFUL"); 816 return ret; 817 } 818 819 hw->adapter_stopped = FALSE; 820 821 /* Setting proper Rx Mode and issue Rx Mode Update command */ 822 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); 823 824 if (dev->data->dev_conf.intr_conf.lsc) { 825 vmxnet3_enable_intr(hw); 826 827 /* 828 * Update link state from device since this won't be 829 * done upon starting with lsc in use. This is done 830 * only after enabling interrupts to avoid any race 831 * where the link state could change without an 832 * interrupt being fired. 833 */ 834 __vmxnet3_dev_link_update(dev, 0); 835 } 836 837 return VMXNET3_SUCCESS; 838 } 839 840 /* 841 * Stop device: disable rx and tx functions to allow for reconfiguring. 842 */ 843 static void 844 vmxnet3_dev_stop(struct rte_eth_dev *dev) 845 { 846 struct rte_eth_link link; 847 struct vmxnet3_hw *hw = dev->data->dev_private; 848 849 PMD_INIT_FUNC_TRACE(); 850 851 if (hw->adapter_stopped == 1) { 852 PMD_INIT_LOG(DEBUG, "Device already closed."); 853 return; 854 } 855 856 /* disable interrupts */ 857 vmxnet3_disable_intr(hw); 858 859 if (dev->data->dev_conf.intr_conf.lsc) { 860 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 861 862 rte_intr_disable(&pci_dev->intr_handle); 863 864 rte_intr_callback_unregister(&pci_dev->intr_handle, 865 vmxnet3_interrupt_handler, dev); 866 } 867 868 /* quiesce the device first */ 869 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 870 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0); 871 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 0); 872 873 /* reset the device */ 874 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 875 PMD_INIT_LOG(DEBUG, "Device reset."); 876 hw->adapter_stopped = 0; 877 878 vmxnet3_dev_clear_queues(dev); 879 880 /* Clear recorded link status */ 881 memset(&link, 0, sizeof(link)); 882 vmxnet3_dev_atomic_write_link_status(dev, &link); 883 } 884 885 /* 886 * Reset and stop device. 887 */ 888 static void 889 vmxnet3_dev_close(struct rte_eth_dev *dev) 890 { 891 struct vmxnet3_hw *hw = dev->data->dev_private; 892 893 PMD_INIT_FUNC_TRACE(); 894 895 vmxnet3_dev_stop(dev); 896 hw->adapter_stopped = 1; 897 } 898 899 static void 900 vmxnet3_hw_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 901 struct UPT1_TxStats *res) 902 { 903 #define VMXNET3_UPDATE_TX_STAT(h, i, f, r) \ 904 ((r)->f = (h)->tqd_start[(i)].stats.f + \ 905 (h)->saved_tx_stats[(i)].f) 906 907 VMXNET3_UPDATE_TX_STAT(hw, q, ucastPktsTxOK, res); 908 VMXNET3_UPDATE_TX_STAT(hw, q, mcastPktsTxOK, res); 909 VMXNET3_UPDATE_TX_STAT(hw, q, bcastPktsTxOK, res); 910 VMXNET3_UPDATE_TX_STAT(hw, q, ucastBytesTxOK, res); 911 VMXNET3_UPDATE_TX_STAT(hw, q, mcastBytesTxOK, res); 912 VMXNET3_UPDATE_TX_STAT(hw, q, bcastBytesTxOK, res); 913 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxError, res); 914 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxDiscard, res); 915 916 #undef VMXNET3_UPDATE_TX_STAT 917 } 918 919 static void 920 vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 921 struct UPT1_RxStats *res) 922 { 923 #define VMXNET3_UPDATE_RX_STAT(h, i, f, r) \ 924 ((r)->f = (h)->rqd_start[(i)].stats.f + \ 925 (h)->saved_rx_stats[(i)].f) 926 927 VMXNET3_UPDATE_RX_STAT(hw, q, ucastPktsRxOK, res); 928 VMXNET3_UPDATE_RX_STAT(hw, q, mcastPktsRxOK, res); 929 VMXNET3_UPDATE_RX_STAT(hw, q, bcastPktsRxOK, res); 930 VMXNET3_UPDATE_RX_STAT(hw, q, ucastBytesRxOK, res); 931 VMXNET3_UPDATE_RX_STAT(hw, q, mcastBytesRxOK, res); 932 VMXNET3_UPDATE_RX_STAT(hw, q, bcastBytesRxOK, res); 933 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res); 934 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res); 935 936 #undef VMXNET3_UPDATE_RX_STATS 937 } 938 939 static void 940 vmxnet3_hw_stats_save(struct vmxnet3_hw *hw) 941 { 942 unsigned int i; 943 944 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 945 946 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); 947 948 for (i = 0; i < hw->num_tx_queues; i++) 949 vmxnet3_hw_tx_stats_get(hw, i, &hw->saved_tx_stats[i]); 950 for (i = 0; i < hw->num_rx_queues; i++) 951 vmxnet3_hw_rx_stats_get(hw, i, &hw->saved_rx_stats[i]); 952 } 953 954 static int 955 vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 956 struct rte_eth_xstat_name *xstats_names, 957 unsigned int n) 958 { 959 unsigned int i, t, count = 0; 960 unsigned int nstats = 961 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 962 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 963 964 if (!xstats_names || n < nstats) 965 return nstats; 966 967 for (i = 0; i < dev->data->nb_rx_queues; i++) { 968 if (!dev->data->rx_queues[i]) 969 continue; 970 971 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 972 snprintf(xstats_names[count].name, 973 sizeof(xstats_names[count].name), 974 "rx_q%u_%s", i, 975 vmxnet3_rxq_stat_strings[t].name); 976 count++; 977 } 978 } 979 980 for (i = 0; i < dev->data->nb_tx_queues; i++) { 981 if (!dev->data->tx_queues[i]) 982 continue; 983 984 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 985 snprintf(xstats_names[count].name, 986 sizeof(xstats_names[count].name), 987 "tx_q%u_%s", i, 988 vmxnet3_txq_stat_strings[t].name); 989 count++; 990 } 991 } 992 993 return count; 994 } 995 996 static int 997 vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 998 unsigned int n) 999 { 1000 unsigned int i, t, count = 0; 1001 unsigned int nstats = 1002 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 1003 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 1004 1005 if (n < nstats) 1006 return nstats; 1007 1008 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1009 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i]; 1010 1011 if (rxq == NULL) 1012 continue; 1013 1014 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 1015 xstats[count].value = *(uint64_t *)(((char *)&rxq->stats) + 1016 vmxnet3_rxq_stat_strings[t].offset); 1017 xstats[count].id = count; 1018 count++; 1019 } 1020 } 1021 1022 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1023 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; 1024 1025 if (txq == NULL) 1026 continue; 1027 1028 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 1029 xstats[count].value = *(uint64_t *)(((char *)&txq->stats) + 1030 vmxnet3_txq_stat_strings[t].offset); 1031 xstats[count].id = count; 1032 count++; 1033 } 1034 } 1035 1036 return count; 1037 } 1038 1039 static int 1040 vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1041 { 1042 unsigned int i; 1043 struct vmxnet3_hw *hw = dev->data->dev_private; 1044 struct UPT1_TxStats txStats; 1045 struct UPT1_RxStats rxStats; 1046 1047 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1048 1049 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); 1050 for (i = 0; i < hw->num_tx_queues; i++) { 1051 vmxnet3_hw_tx_stats_get(hw, i, &txStats); 1052 1053 stats->q_opackets[i] = txStats.ucastPktsTxOK + 1054 txStats.mcastPktsTxOK + 1055 txStats.bcastPktsTxOK; 1056 1057 stats->q_obytes[i] = txStats.ucastBytesTxOK + 1058 txStats.mcastBytesTxOK + 1059 txStats.bcastBytesTxOK; 1060 1061 stats->opackets += stats->q_opackets[i]; 1062 stats->obytes += stats->q_obytes[i]; 1063 stats->oerrors += txStats.pktsTxError + txStats.pktsTxDiscard; 1064 } 1065 1066 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_RX_QUEUES); 1067 for (i = 0; i < hw->num_rx_queues; i++) { 1068 vmxnet3_hw_rx_stats_get(hw, i, &rxStats); 1069 1070 stats->q_ipackets[i] = rxStats.ucastPktsRxOK + 1071 rxStats.mcastPktsRxOK + 1072 rxStats.bcastPktsRxOK; 1073 1074 stats->q_ibytes[i] = rxStats.ucastBytesRxOK + 1075 rxStats.mcastBytesRxOK + 1076 rxStats.bcastBytesRxOK; 1077 1078 stats->ipackets += stats->q_ipackets[i]; 1079 stats->ibytes += stats->q_ibytes[i]; 1080 1081 stats->q_errors[i] = rxStats.pktsRxError; 1082 stats->ierrors += rxStats.pktsRxError; 1083 stats->rx_nombuf += rxStats.pktsRxOutOfBuf; 1084 } 1085 1086 return 0; 1087 } 1088 1089 static void 1090 vmxnet3_dev_info_get(struct rte_eth_dev *dev, 1091 struct rte_eth_dev_info *dev_info) 1092 { 1093 dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1094 1095 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; 1096 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; 1097 dev_info->min_rx_bufsize = 1518 + RTE_PKTMBUF_HEADROOM; 1098 dev_info->max_rx_pktlen = 16384; /* includes CRC, cf MAXFRS register */ 1099 dev_info->speed_capa = ETH_LINK_SPEED_10G; 1100 dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS; 1101 1102 dev_info->default_txconf.txq_flags = ETH_TXQ_FLAGS_NOXSUMSCTP; 1103 dev_info->flow_type_rss_offloads = VMXNET3_RSS_OFFLOAD_ALL; 1104 1105 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 1106 .nb_max = VMXNET3_RX_RING_MAX_SIZE, 1107 .nb_min = VMXNET3_DEF_RX_RING_SIZE, 1108 .nb_align = 1, 1109 }; 1110 1111 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 1112 .nb_max = VMXNET3_TX_RING_MAX_SIZE, 1113 .nb_min = VMXNET3_DEF_TX_RING_SIZE, 1114 .nb_align = 1, 1115 .nb_seg_max = VMXNET3_TX_MAX_SEG, 1116 .nb_mtu_seg_max = VMXNET3_MAX_TXD_PER_PKT, 1117 }; 1118 1119 dev_info->rx_offload_capa = 1120 DEV_RX_OFFLOAD_VLAN_STRIP | 1121 DEV_RX_OFFLOAD_UDP_CKSUM | 1122 DEV_RX_OFFLOAD_TCP_CKSUM | 1123 DEV_RX_OFFLOAD_TCP_LRO; 1124 1125 dev_info->tx_offload_capa = 1126 DEV_TX_OFFLOAD_VLAN_INSERT | 1127 DEV_TX_OFFLOAD_TCP_CKSUM | 1128 DEV_TX_OFFLOAD_UDP_CKSUM | 1129 DEV_TX_OFFLOAD_TCP_TSO; 1130 } 1131 1132 static const uint32_t * 1133 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev) 1134 { 1135 static const uint32_t ptypes[] = { 1136 RTE_PTYPE_L3_IPV4_EXT, 1137 RTE_PTYPE_L3_IPV4, 1138 RTE_PTYPE_UNKNOWN 1139 }; 1140 1141 if (dev->rx_pkt_burst == vmxnet3_recv_pkts) 1142 return ptypes; 1143 return NULL; 1144 } 1145 1146 static void 1147 vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 1148 { 1149 struct vmxnet3_hw *hw = dev->data->dev_private; 1150 1151 ether_addr_copy(mac_addr, (struct ether_addr *)(hw->perm_addr)); 1152 ether_addr_copy(mac_addr, &dev->data->mac_addrs[0]); 1153 vmxnet3_write_mac(hw, mac_addr->addr_bytes); 1154 } 1155 1156 /* return 0 means link status changed, -1 means not changed */ 1157 static int 1158 __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 1159 __rte_unused int wait_to_complete) 1160 { 1161 struct vmxnet3_hw *hw = dev->data->dev_private; 1162 struct rte_eth_link old = { 0 }, link; 1163 uint32_t ret; 1164 1165 memset(&link, 0, sizeof(link)); 1166 vmxnet3_dev_atomic_read_link_status(dev, &old); 1167 1168 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 1169 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1170 1171 if (ret & 0x1) { 1172 link.link_status = ETH_LINK_UP; 1173 link.link_duplex = ETH_LINK_FULL_DUPLEX; 1174 link.link_speed = ETH_SPEED_NUM_10G; 1175 link.link_autoneg = ETH_LINK_SPEED_FIXED; 1176 } 1177 1178 vmxnet3_dev_atomic_write_link_status(dev, &link); 1179 1180 return (old.link_status == link.link_status) ? -1 : 0; 1181 } 1182 1183 static int 1184 vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 1185 { 1186 /* Link status doesn't change for stopped dev */ 1187 if (dev->data->dev_started == 0) 1188 return -1; 1189 1190 return __vmxnet3_dev_link_update(dev, wait_to_complete); 1191 } 1192 1193 /* Updating rxmode through Vmxnet3_DriverShared structure in adapter */ 1194 static void 1195 vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) 1196 { 1197 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1198 1199 if (set) 1200 rxConf->rxMode = rxConf->rxMode | feature; 1201 else 1202 rxConf->rxMode = rxConf->rxMode & (~feature); 1203 1204 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE); 1205 } 1206 1207 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1208 static void 1209 vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev) 1210 { 1211 struct vmxnet3_hw *hw = dev->data->dev_private; 1212 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1213 1214 memset(vf_table, 0, VMXNET3_VFT_TABLE_SIZE); 1215 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 1); 1216 1217 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1218 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1219 } 1220 1221 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1222 static void 1223 vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev) 1224 { 1225 struct vmxnet3_hw *hw = dev->data->dev_private; 1226 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1227 1228 if (dev->data->dev_conf.rxmode.hw_vlan_filter) 1229 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1230 else 1231 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1232 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0); 1233 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1234 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1235 } 1236 1237 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1238 static void 1239 vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev) 1240 { 1241 struct vmxnet3_hw *hw = dev->data->dev_private; 1242 1243 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1); 1244 } 1245 1246 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1247 static void 1248 vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev) 1249 { 1250 struct vmxnet3_hw *hw = dev->data->dev_private; 1251 1252 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 0); 1253 } 1254 1255 /* Enable/disable filter on vlan */ 1256 static int 1257 vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vid, int on) 1258 { 1259 struct vmxnet3_hw *hw = dev->data->dev_private; 1260 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1261 uint32_t *vf_table = rxConf->vfTable; 1262 1263 /* save state for restore */ 1264 if (on) 1265 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1266 else 1267 VMXNET3_CLEAR_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1268 1269 /* don't change active filter if in promiscuous mode */ 1270 if (rxConf->rxMode & VMXNET3_RXM_PROMISC) 1271 return 0; 1272 1273 /* set in hardware */ 1274 if (on) 1275 VMXNET3_SET_VFTABLE_ENTRY(vf_table, vid); 1276 else 1277 VMXNET3_CLEAR_VFTABLE_ENTRY(vf_table, vid); 1278 1279 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1280 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1281 return 0; 1282 } 1283 1284 static int 1285 vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1286 { 1287 struct vmxnet3_hw *hw = dev->data->dev_private; 1288 Vmxnet3_DSDevRead *devRead = &hw->shared->devRead; 1289 uint32_t *vf_table = devRead->rxFilterConf.vfTable; 1290 1291 if (mask & ETH_VLAN_STRIP_MASK) { 1292 if (dev->data->dev_conf.rxmode.hw_vlan_strip) 1293 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 1294 else 1295 devRead->misc.uptFeatures &= ~UPT1_F_RXVLAN; 1296 1297 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1298 VMXNET3_CMD_UPDATE_FEATURE); 1299 } 1300 1301 if (mask & ETH_VLAN_FILTER_MASK) { 1302 if (dev->data->dev_conf.rxmode.hw_vlan_filter) 1303 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1304 else 1305 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1306 1307 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1308 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1309 } 1310 1311 return 0; 1312 } 1313 1314 static void 1315 vmxnet3_process_events(struct rte_eth_dev *dev) 1316 { 1317 struct vmxnet3_hw *hw = dev->data->dev_private; 1318 uint32_t events = hw->shared->ecr; 1319 1320 if (!events) 1321 return; 1322 1323 /* 1324 * ECR bits when written with 1b are cleared. Hence write 1325 * events back to ECR so that the bits which were set will be reset. 1326 */ 1327 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events); 1328 1329 /* Check if link state has changed */ 1330 if (events & VMXNET3_ECR_LINK) { 1331 PMD_DRV_LOG(DEBUG, "Process events: VMXNET3_ECR_LINK event"); 1332 if (vmxnet3_dev_link_update(dev, 0) == 0) 1333 _rte_eth_dev_callback_process(dev, 1334 RTE_ETH_EVENT_INTR_LSC, 1335 NULL, NULL); 1336 } 1337 1338 /* Check if there is an error on xmit/recv queues */ 1339 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 1340 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1341 VMXNET3_CMD_GET_QUEUE_STATUS); 1342 1343 if (hw->tqd_start->status.stopped) 1344 PMD_DRV_LOG(ERR, "tq error 0x%x", 1345 hw->tqd_start->status.error); 1346 1347 if (hw->rqd_start->status.stopped) 1348 PMD_DRV_LOG(ERR, "rq error 0x%x", 1349 hw->rqd_start->status.error); 1350 1351 /* Reset the device */ 1352 /* Have to reset the device */ 1353 } 1354 1355 if (events & VMXNET3_ECR_DIC) 1356 PMD_DRV_LOG(DEBUG, "Device implementation change event."); 1357 1358 if (events & VMXNET3_ECR_DEBUG) 1359 PMD_DRV_LOG(DEBUG, "Debug event generated by device."); 1360 } 1361 1362 static void 1363 vmxnet3_interrupt_handler(void *param) 1364 { 1365 struct rte_eth_dev *dev = param; 1366 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 1367 1368 vmxnet3_process_events(dev); 1369 1370 if (rte_intr_enable(&pci_dev->intr_handle) < 0) 1371 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1372 } 1373 1374 RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd); 1375 RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map); 1376 RTE_PMD_REGISTER_KMOD_DEP(net_vmxnet3, "* igb_uio | uio_pci_generic | vfio-pci"); 1377