1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdio.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <unistd.h> 12 #include <stdarg.h> 13 #include <fcntl.h> 14 #include <inttypes.h> 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 19 #include <rte_interrupts.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_pci.h> 23 #include <bus_pci_driver.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_memory.h> 26 #include <rte_memzone.h> 27 #include <rte_eal.h> 28 #include <rte_alarm.h> 29 #include <rte_ether.h> 30 #include <ethdev_driver.h> 31 #include <ethdev_pci.h> 32 #include <rte_string_fns.h> 33 #include <rte_malloc.h> 34 #include <dev_driver.h> 35 36 #include "base/vmxnet3_defs.h" 37 38 #include "vmxnet3_ring.h" 39 #include "vmxnet3_logs.h" 40 #include "vmxnet3_ethdev.h" 41 42 #define VMXNET3_TX_MAX_SEG UINT8_MAX 43 44 #define VMXNET3_TX_OFFLOAD_CAP \ 45 (RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ 46 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \ 47 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \ 48 RTE_ETH_TX_OFFLOAD_TCP_TSO | \ 49 RTE_ETH_TX_OFFLOAD_MULTI_SEGS) 50 51 #define VMXNET3_RX_OFFLOAD_CAP \ 52 (RTE_ETH_RX_OFFLOAD_VLAN_STRIP | \ 53 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | \ 54 RTE_ETH_RX_OFFLOAD_SCATTER | \ 55 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | \ 56 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | \ 57 RTE_ETH_RX_OFFLOAD_TCP_LRO | \ 58 RTE_ETH_RX_OFFLOAD_RSS_HASH) 59 60 int vmxnet3_segs_dynfield_offset = -1; 61 62 static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev); 63 static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev); 64 static int vmxnet3_dev_configure(struct rte_eth_dev *dev); 65 static int vmxnet3_dev_start(struct rte_eth_dev *dev); 66 static int vmxnet3_dev_stop(struct rte_eth_dev *dev); 67 static int vmxnet3_dev_close(struct rte_eth_dev *dev); 68 static int vmxnet3_dev_reset(struct rte_eth_dev *dev); 69 static void vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set); 70 static int vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev); 71 static int vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev); 72 static int vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev); 73 static int vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev); 74 static int __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 75 int wait_to_complete); 76 static int vmxnet3_dev_link_update(struct rte_eth_dev *dev, 77 int wait_to_complete); 78 static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw); 79 static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev, 80 struct rte_eth_stats *stats); 81 static int vmxnet3_dev_stats_reset(struct rte_eth_dev *dev); 82 static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 83 struct rte_eth_xstat_name *xstats, 84 unsigned int n); 85 static int vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, 86 struct rte_eth_xstat *xstats, unsigned int n); 87 static int vmxnet3_dev_info_get(struct rte_eth_dev *dev, 88 struct rte_eth_dev_info *dev_info); 89 static int vmxnet3_hw_ver_get(struct rte_eth_dev *dev, 90 char *fw_version, size_t fw_size); 91 static const uint32_t * 92 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev, 93 size_t *no_of_elements); 94 static int vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 95 static int vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, 96 uint16_t vid, int on); 97 static int vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 98 static int vmxnet3_mac_addr_set(struct rte_eth_dev *dev, 99 struct rte_ether_addr *mac_addr); 100 static void vmxnet3_process_events(struct rte_eth_dev *dev); 101 static void vmxnet3_interrupt_handler(void *param); 102 static int 103 vmxnet3_rss_reta_update(struct rte_eth_dev *dev, 104 struct rte_eth_rss_reta_entry64 *reta_conf, 105 uint16_t reta_size); 106 static int 107 vmxnet3_rss_reta_query(struct rte_eth_dev *dev, 108 struct rte_eth_rss_reta_entry64 *reta_conf, 109 uint16_t reta_size); 110 111 static int vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 112 uint16_t queue_id); 113 static int vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 114 uint16_t queue_id); 115 116 /* 117 * The set of PCI devices this driver supports 118 */ 119 #define VMWARE_PCI_VENDOR_ID 0x15AD 120 #define VMWARE_DEV_ID_VMXNET3 0x07B0 121 static const struct rte_pci_id pci_id_vmxnet3_map[] = { 122 { RTE_PCI_DEVICE(VMWARE_PCI_VENDOR_ID, VMWARE_DEV_ID_VMXNET3) }, 123 { .vendor_id = 0, /* sentinel */ }, 124 }; 125 126 static const struct eth_dev_ops vmxnet3_eth_dev_ops = { 127 .dev_configure = vmxnet3_dev_configure, 128 .dev_start = vmxnet3_dev_start, 129 .dev_stop = vmxnet3_dev_stop, 130 .dev_close = vmxnet3_dev_close, 131 .dev_reset = vmxnet3_dev_reset, 132 .link_update = vmxnet3_dev_link_update, 133 .promiscuous_enable = vmxnet3_dev_promiscuous_enable, 134 .promiscuous_disable = vmxnet3_dev_promiscuous_disable, 135 .allmulticast_enable = vmxnet3_dev_allmulticast_enable, 136 .allmulticast_disable = vmxnet3_dev_allmulticast_disable, 137 .mac_addr_set = vmxnet3_mac_addr_set, 138 .mtu_set = vmxnet3_dev_mtu_set, 139 .stats_get = vmxnet3_dev_stats_get, 140 .stats_reset = vmxnet3_dev_stats_reset, 141 .xstats_get = vmxnet3_dev_xstats_get, 142 .xstats_get_names = vmxnet3_dev_xstats_get_names, 143 .dev_infos_get = vmxnet3_dev_info_get, 144 .fw_version_get = vmxnet3_hw_ver_get, 145 .dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get, 146 .vlan_filter_set = vmxnet3_dev_vlan_filter_set, 147 .vlan_offload_set = vmxnet3_dev_vlan_offload_set, 148 .rx_queue_setup = vmxnet3_dev_rx_queue_setup, 149 .rx_queue_release = vmxnet3_dev_rx_queue_release, 150 .rx_queue_intr_enable = vmxnet3_dev_rx_queue_intr_enable, 151 .rx_queue_intr_disable = vmxnet3_dev_rx_queue_intr_disable, 152 .tx_queue_setup = vmxnet3_dev_tx_queue_setup, 153 .tx_queue_release = vmxnet3_dev_tx_queue_release, 154 .reta_update = vmxnet3_rss_reta_update, 155 .reta_query = vmxnet3_rss_reta_query, 156 }; 157 158 struct vmxnet3_xstats_name_off { 159 char name[RTE_ETH_XSTATS_NAME_SIZE]; 160 unsigned int offset; 161 }; 162 163 /* tx_qX_ is prepended to the name string here */ 164 static const struct vmxnet3_xstats_name_off vmxnet3_txq_stat_strings[] = { 165 {"drop_total", offsetof(struct vmxnet3_txq_stats, drop_total)}, 166 {"drop_too_many_segs", offsetof(struct vmxnet3_txq_stats, drop_too_many_segs)}, 167 {"drop_tso", offsetof(struct vmxnet3_txq_stats, drop_tso)}, 168 {"tx_ring_full", offsetof(struct vmxnet3_txq_stats, tx_ring_full)}, 169 }; 170 171 /* rx_qX_ is prepended to the name string here */ 172 static const struct vmxnet3_xstats_name_off vmxnet3_rxq_stat_strings[] = { 173 {"drop_total", offsetof(struct vmxnet3_rxq_stats, drop_total)}, 174 {"drop_err", offsetof(struct vmxnet3_rxq_stats, drop_err)}, 175 {"drop_fcs", offsetof(struct vmxnet3_rxq_stats, drop_fcs)}, 176 {"rx_buf_alloc_failure", offsetof(struct vmxnet3_rxq_stats, rx_buf_alloc_failure)}, 177 }; 178 179 static const struct rte_memzone * 180 gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size, 181 const char *post_string, int socket_id, 182 uint16_t align, bool reuse) 183 { 184 char z_name[RTE_MEMZONE_NAMESIZE]; 185 const struct rte_memzone *mz; 186 187 snprintf(z_name, sizeof(z_name), "eth_p%d_%s", 188 dev->data->port_id, post_string); 189 190 mz = rte_memzone_lookup(z_name); 191 if (!reuse) { 192 rte_memzone_free(mz); 193 return rte_memzone_reserve_aligned(z_name, size, socket_id, 194 RTE_MEMZONE_IOVA_CONTIG, align); 195 } 196 197 if (mz) 198 return mz; 199 200 return rte_memzone_reserve_aligned(z_name, size, socket_id, 201 RTE_MEMZONE_IOVA_CONTIG, align); 202 } 203 204 /* 205 * Enable the given interrupt 206 */ 207 static void 208 vmxnet3_enable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx) 209 { 210 PMD_INIT_FUNC_TRACE(); 211 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 0); 212 } 213 214 /* 215 * Disable the given interrupt 216 */ 217 static void 218 vmxnet3_disable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx) 219 { 220 PMD_INIT_FUNC_TRACE(); 221 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 1); 222 } 223 224 /* 225 * Simple helper to get intrCtrl and eventIntrIdx based on config and hw version 226 */ 227 static void 228 vmxnet3_get_intr_ctrl_ev(struct vmxnet3_hw *hw, 229 uint8 **out_eventIntrIdx, 230 uint32 **out_intrCtrl) 231 { 232 233 if (VMXNET3_VERSION_GE_6(hw) && hw->queuesExtEnabled) { 234 *out_eventIntrIdx = &hw->shared->devReadExt.intrConfExt.eventIntrIdx; 235 *out_intrCtrl = &hw->shared->devReadExt.intrConfExt.intrCtrl; 236 } else { 237 *out_eventIntrIdx = &hw->shared->devRead.intrConf.eventIntrIdx; 238 *out_intrCtrl = &hw->shared->devRead.intrConf.intrCtrl; 239 } 240 } 241 242 /* 243 * Disable all intrs used by the device 244 */ 245 static void 246 vmxnet3_disable_all_intrs(struct vmxnet3_hw *hw) 247 { 248 int i; 249 uint8 *eventIntrIdx; 250 uint32 *intrCtrl; 251 252 PMD_INIT_FUNC_TRACE(); 253 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 254 255 *intrCtrl |= rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 256 257 for (i = 0; i < hw->intr.num_intrs; i++) 258 vmxnet3_disable_intr(hw, i); 259 } 260 261 #ifndef RTE_EXEC_ENV_FREEBSD 262 /* 263 * Enable all intrs used by the device 264 */ 265 static void 266 vmxnet3_enable_all_intrs(struct vmxnet3_hw *hw) 267 { 268 uint8 *eventIntrIdx; 269 uint32 *intrCtrl; 270 271 PMD_INIT_FUNC_TRACE(); 272 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 273 274 *intrCtrl &= rte_cpu_to_le_32(~VMXNET3_IC_DISABLE_ALL); 275 276 if (hw->intr.lsc_only) { 277 vmxnet3_enable_intr(hw, *eventIntrIdx); 278 } else { 279 int i; 280 281 for (i = 0; i < hw->intr.num_intrs; i++) 282 vmxnet3_enable_intr(hw, i); 283 } 284 } 285 #endif 286 287 /* 288 * Gets tx data ring descriptor size. 289 */ 290 static uint16_t 291 eth_vmxnet3_txdata_get(struct vmxnet3_hw *hw) 292 { 293 uint16 txdata_desc_size; 294 295 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 296 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 297 txdata_desc_size = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 298 299 return (txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE || 300 txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE || 301 txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK) ? 302 sizeof(struct Vmxnet3_TxDataDesc) : txdata_desc_size; 303 } 304 305 static int 306 eth_vmxnet3_setup_capabilities(struct vmxnet3_hw *hw, 307 struct rte_eth_dev *eth_dev) 308 { 309 uint32_t dcr, ptcr, value; 310 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 311 312 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 313 VMXNET3_CMD_GET_MAX_CAPABILITIES); 314 value = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 315 hw->max_capabilities[0] = value; 316 dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_DCR); 317 hw->DCR_capabilities[0] = dcr; 318 hw->used_DCR_capabilities[0] = 0; 319 ptcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_PTCR); 320 hw->PTCR_capabilities[0] = ptcr; 321 hw->used_PTCR_capabilities[0] = 0; 322 323 if (hw->uptv2_enabled && !(ptcr & (1 << VMXNET3_DCR_ERROR))) { 324 PMD_DRV_LOG(NOTICE, "UPTv2 enabled"); 325 hw->used_PTCR_capabilities[0] = ptcr; 326 } else { 327 /* Use all DCR capabilities, but disable large bar */ 328 hw->used_DCR_capabilities[0] = dcr & 329 (~(1UL << VMXNET3_CAP_LARGE_BAR)); 330 PMD_DRV_LOG(NOTICE, "UPTv2 disabled"); 331 } 332 if (hw->DCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 333 hw->PTCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 334 if (hw->uptv2_enabled) { 335 hw->used_PTCR_capabilities[0] |= 336 (1UL << VMXNET3_CAP_OOORX_COMP); 337 } 338 } 339 if (hw->used_PTCR_capabilities[0]) { 340 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR, 341 hw->used_PTCR_capabilities[0]); 342 } else if (hw->used_DCR_capabilities[0]) { 343 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR, 344 hw->used_DCR_capabilities[0]); 345 } 346 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 347 dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 348 hw->used_DCR_capabilities[0] = dcr; 349 PMD_DRV_LOG(DEBUG, "Dev " PCI_PRI_FMT ", vmxnet3 v%d, UPT enabled: %s, " 350 "DCR0=0x%08x, used DCR=0x%08x, " 351 "PTCR=0x%08x, used PTCR=0x%08x", 352 pci_dev->addr.domain, pci_dev->addr.bus, 353 pci_dev->addr.devid, pci_dev->addr.function, hw->version, 354 hw->uptv2_enabled ? "true" : "false", 355 hw->DCR_capabilities[0], hw->used_DCR_capabilities[0], 356 hw->PTCR_capabilities[0], hw->used_PTCR_capabilities[0]); 357 return 0; 358 } 359 360 /* 361 * It returns 0 on success. 362 */ 363 static int 364 eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) 365 { 366 struct rte_pci_device *pci_dev; 367 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 368 uint32_t mac_hi, mac_lo, ver; 369 struct rte_eth_link link; 370 static const struct rte_mbuf_dynfield vmxnet3_segs_dynfield_desc = { 371 .name = VMXNET3_SEGS_DYNFIELD_NAME, 372 .size = sizeof(vmxnet3_segs_dynfield_t), 373 .align = alignof(vmxnet3_segs_dynfield_t), 374 }; 375 376 PMD_INIT_FUNC_TRACE(); 377 378 eth_dev->dev_ops = &vmxnet3_eth_dev_ops; 379 eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts; 380 eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts; 381 eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts; 382 eth_dev->rx_queue_count = vmxnet3_dev_rx_queue_count; 383 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 384 385 /* extra mbuf field is required to guess MSS */ 386 vmxnet3_segs_dynfield_offset = 387 rte_mbuf_dynfield_register(&vmxnet3_segs_dynfield_desc); 388 if (vmxnet3_segs_dynfield_offset < 0) { 389 PMD_INIT_LOG(ERR, "Cannot register mbuf field."); 390 return -rte_errno; 391 } 392 393 /* 394 * for secondary processes, we don't initialize any further as primary 395 * has already done this work. 396 */ 397 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 398 return 0; 399 400 rte_eth_copy_pci_info(eth_dev, pci_dev); 401 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 402 403 /* Vendor and Device ID need to be set before init of shared code */ 404 hw->device_id = pci_dev->id.device_id; 405 hw->vendor_id = pci_dev->id.vendor_id; 406 hw->hw_addr0 = (void *)pci_dev->mem_resource[0].addr; 407 hw->hw_addr1 = (void *)pci_dev->mem_resource[1].addr; 408 409 hw->num_rx_queues = 1; 410 hw->num_tx_queues = 1; 411 hw->bufs_per_pkt = 1; 412 413 /* Check h/w version compatibility with driver. */ 414 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_VRRS); 415 416 if (ver & (1 << VMXNET3_REV_7)) { 417 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 418 1 << VMXNET3_REV_7); 419 hw->version = VMXNET3_REV_7 + 1; 420 } else if (ver & (1 << VMXNET3_REV_6)) { 421 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 422 1 << VMXNET3_REV_6); 423 hw->version = VMXNET3_REV_6 + 1; 424 } else if (ver & (1 << VMXNET3_REV_5)) { 425 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 426 1 << VMXNET3_REV_5); 427 hw->version = VMXNET3_REV_5 + 1; 428 } else if (ver & (1 << VMXNET3_REV_4)) { 429 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 430 1 << VMXNET3_REV_4); 431 hw->version = VMXNET3_REV_4 + 1; 432 } else if (ver & (1 << VMXNET3_REV_3)) { 433 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 434 1 << VMXNET3_REV_3); 435 hw->version = VMXNET3_REV_3 + 1; 436 } else if (ver & (1 << VMXNET3_REV_2)) { 437 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 438 1 << VMXNET3_REV_2); 439 hw->version = VMXNET3_REV_2 + 1; 440 } else if (ver & (1 << VMXNET3_REV_1)) { 441 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 442 1 << VMXNET3_REV_1); 443 hw->version = VMXNET3_REV_1 + 1; 444 } else { 445 PMD_INIT_LOG(ERR, "Incompatible hardware version: %d", ver); 446 return -EIO; 447 } 448 449 PMD_INIT_LOG(INFO, "Using device v%d", hw->version); 450 451 /* Check UPT version compatibility with driver. */ 452 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_UVRS); 453 PMD_INIT_LOG(DEBUG, "UPT hardware version : %d", ver); 454 if (ver & 0x1) 455 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_UVRS, 1); 456 else { 457 PMD_INIT_LOG(ERR, "Incompatible UPT version."); 458 return -EIO; 459 } 460 461 if (VMXNET3_VERSION_GE_7(hw)) { 462 /* start with UPTv2 enabled to avoid ESXi issues */ 463 hw->uptv2_enabled = TRUE; 464 eth_vmxnet3_setup_capabilities(hw, eth_dev); 465 } 466 467 if (hw->used_DCR_capabilities[0] & (1 << VMXNET3_CAP_LARGE_BAR)) { 468 hw->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 469 hw->rx_prod_offset[0] = VMXNET3_REG_LB_RXPROD; 470 hw->rx_prod_offset[1] = VMXNET3_REG_LB_RXPROD2; 471 } else { 472 hw->tx_prod_offset = VMXNET3_REG_TXPROD; 473 hw->rx_prod_offset[0] = VMXNET3_REG_RXPROD; 474 hw->rx_prod_offset[1] = VMXNET3_REG_RXPROD2; 475 } 476 477 /* Getting MAC Address */ 478 mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL); 479 mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH); 480 memcpy(hw->perm_addr, &mac_lo, 4); 481 memcpy(hw->perm_addr + 4, &mac_hi, 2); 482 483 /* Allocate memory for storing MAC addresses */ 484 eth_dev->data->mac_addrs = rte_zmalloc("vmxnet3", RTE_ETHER_ADDR_LEN * 485 VMXNET3_MAX_MAC_ADDRS, 0); 486 if (eth_dev->data->mac_addrs == NULL) { 487 PMD_INIT_LOG(ERR, 488 "Failed to allocate %d bytes needed to store MAC addresses", 489 RTE_ETHER_ADDR_LEN * VMXNET3_MAX_MAC_ADDRS); 490 return -ENOMEM; 491 } 492 /* Copy the permanent MAC address */ 493 rte_ether_addr_copy((struct rte_ether_addr *)hw->perm_addr, 494 ð_dev->data->mac_addrs[0]); 495 496 PMD_INIT_LOG(DEBUG, "MAC Address : " RTE_ETHER_ADDR_PRT_FMT, 497 hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2], 498 hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]); 499 500 /* Put device in Quiesce Mode */ 501 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 502 503 /* allow untagged pkts */ 504 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, 0); 505 506 hw->txdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 507 eth_vmxnet3_txdata_get(hw) : sizeof(struct Vmxnet3_TxDataDesc); 508 509 hw->rxdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 510 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 511 RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) == 512 hw->rxdata_desc_size); 513 514 /* clear shadow stats */ 515 memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats)); 516 memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats)); 517 518 /* clear snapshot stats */ 519 memset(hw->snapshot_tx_stats, 0, sizeof(hw->snapshot_tx_stats)); 520 memset(hw->snapshot_rx_stats, 0, sizeof(hw->snapshot_rx_stats)); 521 522 /* set the initial link status */ 523 memset(&link, 0, sizeof(link)); 524 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 525 link.link_speed = RTE_ETH_SPEED_NUM_10G; 526 link.link_autoneg = RTE_ETH_LINK_FIXED; 527 rte_eth_linkstatus_set(eth_dev, &link); 528 529 return 0; 530 } 531 532 static int 533 eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev) 534 { 535 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 536 537 PMD_INIT_FUNC_TRACE(); 538 539 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 540 return 0; 541 542 if (hw->adapter_stopped == 0) { 543 PMD_INIT_LOG(DEBUG, "Device has not been closed."); 544 return -EBUSY; 545 } 546 547 return 0; 548 } 549 550 static int eth_vmxnet3_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 551 struct rte_pci_device *pci_dev) 552 { 553 return rte_eth_dev_pci_generic_probe(pci_dev, 554 sizeof(struct vmxnet3_hw), eth_vmxnet3_dev_init); 555 } 556 557 static int eth_vmxnet3_pci_remove(struct rte_pci_device *pci_dev) 558 { 559 return rte_eth_dev_pci_generic_remove(pci_dev, eth_vmxnet3_dev_uninit); 560 } 561 562 static struct rte_pci_driver rte_vmxnet3_pmd = { 563 .id_table = pci_id_vmxnet3_map, 564 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 565 .probe = eth_vmxnet3_pci_probe, 566 .remove = eth_vmxnet3_pci_remove, 567 }; 568 569 static void 570 vmxnet3_alloc_intr_resources(struct rte_eth_dev *dev) 571 { 572 struct vmxnet3_hw *hw = dev->data->dev_private; 573 uint32_t cfg; 574 int nvec = 1; /* for link event */ 575 576 /* intr settings */ 577 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 578 VMXNET3_CMD_GET_CONF_INTR); 579 cfg = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 580 hw->intr.type = cfg & 0x3; 581 hw->intr.mask_mode = (cfg >> 2) & 0x3; 582 583 if (hw->intr.type == VMXNET3_IT_AUTO) 584 hw->intr.type = VMXNET3_IT_MSIX; 585 586 if (hw->intr.type == VMXNET3_IT_MSIX) { 587 /* only support shared tx/rx intr */ 588 if (hw->num_tx_queues != hw->num_rx_queues) 589 goto msix_err; 590 591 nvec += hw->num_rx_queues; 592 hw->intr.num_intrs = nvec; 593 return; 594 } 595 596 msix_err: 597 /* the tx/rx queue interrupt will be disabled */ 598 hw->intr.num_intrs = 2; 599 hw->intr.lsc_only = TRUE; 600 PMD_INIT_LOG(INFO, "Enabled MSI-X with %d vectors", hw->intr.num_intrs); 601 } 602 603 static int 604 vmxnet3_dev_configure(struct rte_eth_dev *dev) 605 { 606 const struct rte_memzone *mz; 607 struct vmxnet3_hw *hw = dev->data->dev_private; 608 size_t size; 609 610 PMD_INIT_FUNC_TRACE(); 611 612 if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) 613 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; 614 615 if (!VMXNET3_VERSION_GE_6(hw)) { 616 if (!rte_is_power_of_2(dev->data->nb_rx_queues)) { 617 PMD_INIT_LOG(ERR, 618 "ERROR: Number of rx queues not power of 2"); 619 return -EINVAL; 620 } 621 } 622 623 /* At this point, the number of queues requested has already 624 * been validated against dev_infos max queues by EAL 625 */ 626 if (dev->data->nb_rx_queues > VMXNET3_MAX_RX_QUEUES || 627 dev->data->nb_tx_queues > VMXNET3_MAX_TX_QUEUES) { 628 hw->queuesExtEnabled = 1; 629 } else { 630 hw->queuesExtEnabled = 0; 631 } 632 633 size = dev->data->nb_rx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 634 dev->data->nb_tx_queues * sizeof(struct Vmxnet3_RxQueueDesc); 635 636 if (size > UINT16_MAX) 637 return -EINVAL; 638 639 hw->num_rx_queues = (uint8_t)dev->data->nb_rx_queues; 640 hw->num_tx_queues = (uint8_t)dev->data->nb_tx_queues; 641 642 /* 643 * Allocate a memzone for Vmxnet3_DriverShared - Vmxnet3_DSDevRead 644 * on current socket 645 */ 646 mz = gpa_zone_reserve(dev, sizeof(struct Vmxnet3_DriverShared), 647 "shared", rte_socket_id(), 8, 1); 648 649 if (mz == NULL) { 650 PMD_INIT_LOG(ERR, "ERROR: Creating shared zone"); 651 return -ENOMEM; 652 } 653 memset(mz->addr, 0, mz->len); 654 655 hw->shared = mz->addr; 656 hw->sharedPA = mz->iova; 657 658 /* 659 * Allocate a memzone for Vmxnet3_RxQueueDesc - Vmxnet3_TxQueueDesc 660 * on current socket. 661 * 662 * We cannot reuse this memzone from previous allocation as its size 663 * depends on the number of tx and rx queues, which could be different 664 * from one config to another. 665 */ 666 mz = gpa_zone_reserve(dev, size, "queuedesc", rte_socket_id(), 667 VMXNET3_QUEUE_DESC_ALIGN, 0); 668 if (mz == NULL) { 669 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); 670 return -ENOMEM; 671 } 672 memset(mz->addr, 0, mz->len); 673 674 hw->tqd_start = (Vmxnet3_TxQueueDesc *)mz->addr; 675 hw->rqd_start = (Vmxnet3_RxQueueDesc *)(hw->tqd_start + hw->num_tx_queues); 676 677 hw->queueDescPA = mz->iova; 678 hw->queue_desc_len = (uint16_t)size; 679 680 if (dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 681 /* Allocate memory structure for UPT1_RSSConf and configure */ 682 mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf), 683 "rss_conf", rte_socket_id(), 684 RTE_CACHE_LINE_SIZE, 1); 685 if (mz == NULL) { 686 PMD_INIT_LOG(ERR, 687 "ERROR: Creating rss_conf structure zone"); 688 return -ENOMEM; 689 } 690 memset(mz->addr, 0, mz->len); 691 692 hw->rss_conf = mz->addr; 693 hw->rss_confPA = mz->iova; 694 } 695 696 vmxnet3_alloc_intr_resources(dev); 697 698 return 0; 699 } 700 701 static void 702 vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr) 703 { 704 uint32_t val; 705 706 PMD_INIT_LOG(DEBUG, 707 "Writing MAC Address : " RTE_ETHER_ADDR_PRT_FMT, 708 addr[0], addr[1], addr[2], 709 addr[3], addr[4], addr[5]); 710 711 memcpy(&val, addr, 4); 712 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACL, val); 713 714 memcpy(&val, addr + 4, 2); 715 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val); 716 } 717 718 /* 719 * Configure the hardware to generate MSI-X interrupts. 720 * If setting up MSIx fails, try setting up MSI (only 1 interrupt vector 721 * which will be disabled to allow lsc to work). 722 * 723 * Returns 0 on success and -1 otherwise. 724 */ 725 static int 726 vmxnet3_configure_msix(struct rte_eth_dev *dev) 727 { 728 struct vmxnet3_hw *hw = dev->data->dev_private; 729 struct rte_intr_handle *intr_handle = dev->intr_handle; 730 uint16_t intr_vector; 731 int i; 732 733 hw->intr.event_intr_idx = 0; 734 735 /* only vfio-pci driver can support interrupt mode. */ 736 if (!rte_intr_cap_multiple(intr_handle) || 737 dev->data->dev_conf.intr_conf.rxq == 0) 738 return -1; 739 740 intr_vector = dev->data->nb_rx_queues; 741 if (intr_vector > MAX_RX_QUEUES(hw)) { 742 PMD_INIT_LOG(ERR, "At most %d intr queues supported", 743 MAX_RX_QUEUES(hw)); 744 return -ENOTSUP; 745 } 746 747 if (rte_intr_efd_enable(intr_handle, intr_vector)) { 748 PMD_INIT_LOG(ERR, "Failed to enable fastpath event fd"); 749 return -1; 750 } 751 752 if (rte_intr_dp_is_en(intr_handle)) { 753 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec", 754 dev->data->nb_rx_queues)) { 755 PMD_INIT_LOG(ERR, "Failed to allocate %d Rx queues intr_vec", 756 dev->data->nb_rx_queues); 757 rte_intr_efd_disable(intr_handle); 758 return -ENOMEM; 759 } 760 } 761 762 if (!rte_intr_allow_others(intr_handle) && 763 dev->data->dev_conf.intr_conf.lsc != 0) { 764 PMD_INIT_LOG(ERR, "not enough intr vector to support both Rx interrupt and LSC"); 765 rte_intr_vec_list_free(intr_handle); 766 rte_intr_efd_disable(intr_handle); 767 return -1; 768 } 769 770 /* if we cannot allocate one MSI-X vector per queue, don't enable 771 * interrupt mode. 772 */ 773 if (hw->intr.num_intrs != 774 (rte_intr_nb_efd_get(intr_handle) + 1)) { 775 PMD_INIT_LOG(ERR, "Device configured with %d Rx intr vectors, expecting %d", 776 hw->intr.num_intrs, 777 rte_intr_nb_efd_get(intr_handle) + 1); 778 rte_intr_vec_list_free(intr_handle); 779 rte_intr_efd_disable(intr_handle); 780 return -1; 781 } 782 783 for (i = 0; i < dev->data->nb_rx_queues; i++) 784 if (rte_intr_vec_list_index_set(intr_handle, i, i + 1)) 785 return -rte_errno; 786 787 for (i = 0; i < hw->intr.num_intrs; i++) 788 hw->intr.mod_levels[i] = UPT1_IML_ADAPTIVE; 789 790 PMD_INIT_LOG(INFO, "intr type %u, mode %u, %u vectors allocated", 791 hw->intr.type, hw->intr.mask_mode, hw->intr.num_intrs); 792 793 return 0; 794 } 795 796 static int 797 vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev) 798 { 799 struct vmxnet3_hw *hw = dev->data->dev_private; 800 Vmxnet3_DriverShared *shared = hw->shared; 801 Vmxnet3_CmdInfo *cmdInfo; 802 struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES]; 803 uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES]; 804 uint32_t num, i, j, size; 805 806 if (hw->memRegsPA == 0) { 807 const struct rte_memzone *mz; 808 809 size = sizeof(Vmxnet3_MemRegs) + 810 (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) * 811 sizeof(Vmxnet3_MemoryRegion); 812 813 mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8, 814 1); 815 if (mz == NULL) { 816 PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone"); 817 return -ENOMEM; 818 } 819 memset(mz->addr, 0, mz->len); 820 hw->memRegs = mz->addr; 821 hw->memRegsPA = mz->iova; 822 } 823 824 num = hw->num_rx_queues; 825 826 for (i = 0; i < num; i++) { 827 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 828 829 mp[i] = rxq->mp; 830 index[i] = 1 << i; 831 } 832 833 /* 834 * The same mempool could be used by multiple queues. In such a case, 835 * remove duplicate mempool entries. Only one entry is kept with 836 * bitmask indicating queues that are using this mempool. 837 */ 838 for (i = 1; i < num; i++) { 839 for (j = 0; j < i; j++) { 840 if (mp[i] == mp[j]) { 841 mp[i] = NULL; 842 index[j] |= 1 << i; 843 break; 844 } 845 } 846 } 847 848 j = 0; 849 for (i = 0; i < num; i++) { 850 if (mp[i] == NULL) 851 continue; 852 853 Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j]; 854 855 mr->startPA = 856 (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->iova; 857 mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ? 858 STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX; 859 mr->txQueueBits = index[i]; 860 mr->rxQueueBits = index[i]; 861 862 PMD_INIT_LOG(INFO, 863 "index: %u startPA: %" PRIu64 " length: %u, " 864 "rxBits: %x", 865 j, mr->startPA, mr->length, mr->rxQueueBits); 866 j++; 867 } 868 hw->memRegs->numRegs = j; 869 PMD_INIT_LOG(INFO, "numRegs: %u", j); 870 871 size = sizeof(Vmxnet3_MemRegs) + 872 (j - 1) * sizeof(Vmxnet3_MemoryRegion); 873 874 cmdInfo = &shared->cu.cmdInfo; 875 cmdInfo->varConf.confVer = 1; 876 cmdInfo->varConf.confLen = size; 877 cmdInfo->varConf.confPA = hw->memRegsPA; 878 879 return 0; 880 } 881 882 static int 883 vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) 884 { 885 struct rte_eth_conf port_conf = dev->data->dev_conf; 886 struct vmxnet3_hw *hw = dev->data->dev_private; 887 struct rte_intr_handle *intr_handle = dev->intr_handle; 888 uint32_t mtu = dev->data->mtu; 889 Vmxnet3_DriverShared *shared = hw->shared; 890 Vmxnet3_DSDevRead *devRead = &shared->devRead; 891 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 892 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 893 uint32_t i; 894 int ret; 895 896 hw->mtu = mtu; 897 898 shared->magic = VMXNET3_REV1_MAGIC; 899 devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM; 900 901 /* Setting up Guest OS information */ 902 devRead->misc.driverInfo.gos.gosBits = sizeof(void *) == 4 ? 903 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64; 904 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 905 devRead->misc.driverInfo.vmxnet3RevSpt = 1; 906 devRead->misc.driverInfo.uptVerSpt = 1; 907 908 devRead->misc.mtu = rte_le_to_cpu_32(mtu); 909 devRead->misc.queueDescPA = hw->queueDescPA; 910 devRead->misc.queueDescLen = hw->queue_desc_len; 911 devRead->misc.numTxQueues = hw->num_tx_queues; 912 devRead->misc.numRxQueues = hw->num_rx_queues; 913 914 for (i = 0; i < hw->num_tx_queues; i++) { 915 Vmxnet3_TxQueueDesc *tqd = &hw->tqd_start[i]; 916 vmxnet3_tx_queue_t *txq = dev->data->tx_queues[i]; 917 918 txq->shared = &hw->tqd_start[i]; 919 920 tqd->ctrl.txNumDeferred = 0; 921 tqd->ctrl.txThreshold = 1; 922 tqd->conf.txRingBasePA = txq->cmd_ring.basePA; 923 tqd->conf.compRingBasePA = txq->comp_ring.basePA; 924 tqd->conf.dataRingBasePA = txq->data_ring.basePA; 925 926 tqd->conf.txRingSize = txq->cmd_ring.size; 927 tqd->conf.compRingSize = txq->comp_ring.size; 928 tqd->conf.dataRingSize = txq->data_ring.size; 929 tqd->conf.txDataRingDescSize = txq->txdata_desc_size; 930 931 if (hw->intr.lsc_only) 932 tqd->conf.intrIdx = 1; 933 else 934 tqd->conf.intrIdx = 935 rte_intr_vec_list_index_get(intr_handle, 936 i); 937 tqd->status.stopped = TRUE; 938 tqd->status.error = 0; 939 memset(&tqd->stats, 0, sizeof(tqd->stats)); 940 } 941 942 for (i = 0; i < hw->num_rx_queues; i++) { 943 Vmxnet3_RxQueueDesc *rqd = &hw->rqd_start[i]; 944 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 945 946 rxq->shared = &hw->rqd_start[i]; 947 948 rqd->conf.rxRingBasePA[0] = rxq->cmd_ring[0].basePA; 949 rqd->conf.rxRingBasePA[1] = rxq->cmd_ring[1].basePA; 950 rqd->conf.compRingBasePA = rxq->comp_ring.basePA; 951 952 rqd->conf.rxRingSize[0] = rxq->cmd_ring[0].size; 953 rqd->conf.rxRingSize[1] = rxq->cmd_ring[1].size; 954 rqd->conf.compRingSize = rxq->comp_ring.size; 955 956 if (VMXNET3_VERSION_GE_3(hw)) { 957 rqd->conf.rxDataRingBasePA = rxq->data_ring.basePA; 958 rqd->conf.rxDataRingDescSize = rxq->data_desc_size; 959 } 960 961 if (hw->intr.lsc_only) 962 rqd->conf.intrIdx = 1; 963 else 964 rqd->conf.intrIdx = 965 rte_intr_vec_list_index_get(intr_handle, 966 i); 967 rqd->status.stopped = TRUE; 968 rqd->status.error = 0; 969 memset(&rqd->stats, 0, sizeof(rqd->stats)); 970 } 971 972 /* intr settings */ 973 if (VMXNET3_VERSION_GE_6(hw) && hw->queuesExtEnabled) { 974 devReadExt->intrConfExt.autoMask = hw->intr.mask_mode == 975 VMXNET3_IMM_AUTO; 976 devReadExt->intrConfExt.numIntrs = hw->intr.num_intrs; 977 for (i = 0; i < hw->intr.num_intrs; i++) 978 devReadExt->intrConfExt.modLevels[i] = 979 hw->intr.mod_levels[i]; 980 981 devReadExt->intrConfExt.eventIntrIdx = hw->intr.event_intr_idx; 982 devReadExt->intrConfExt.intrCtrl |= 983 rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 984 } else { 985 devRead->intrConf.autoMask = hw->intr.mask_mode == 986 VMXNET3_IMM_AUTO; 987 devRead->intrConf.numIntrs = hw->intr.num_intrs; 988 for (i = 0; i < hw->intr.num_intrs; i++) 989 devRead->intrConf.modLevels[i] = hw->intr.mod_levels[i]; 990 991 devRead->intrConf.eventIntrIdx = hw->intr.event_intr_idx; 992 devRead->intrConf.intrCtrl |= rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 993 } 994 995 /* RxMode set to 0 of VMXNET3_RXM_xxx */ 996 devRead->rxFilterConf.rxMode = 0; 997 998 /* Setting up feature flags */ 999 if (rx_offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 1000 devRead->misc.uptFeatures |= VMXNET3_F_RXCSUM; 1001 1002 if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) { 1003 devRead->misc.uptFeatures |= VMXNET3_F_LRO; 1004 devRead->misc.maxNumRxSG = 0; 1005 } 1006 1007 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 1008 ret = vmxnet3_rss_configure(dev); 1009 if (ret != VMXNET3_SUCCESS) 1010 return ret; 1011 1012 devRead->misc.uptFeatures |= VMXNET3_F_RSS; 1013 devRead->rssConfDesc.confVer = 1; 1014 devRead->rssConfDesc.confLen = sizeof(struct VMXNET3_RSSConf); 1015 devRead->rssConfDesc.confPA = hw->rss_confPA; 1016 } 1017 1018 ret = vmxnet3_dev_vlan_offload_set(dev, 1019 RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK); 1020 if (ret) 1021 return ret; 1022 1023 vmxnet3_write_mac(hw, dev->data->mac_addrs->addr_bytes); 1024 1025 return VMXNET3_SUCCESS; 1026 } 1027 1028 static void 1029 vmxnet3_init_bufsize(struct vmxnet3_hw *hw) 1030 { 1031 struct Vmxnet3_DriverShared *shared = hw->shared; 1032 union Vmxnet3_CmdInfo *cmd_info = &shared->cu.cmdInfo; 1033 1034 if (!VMXNET3_VERSION_GE_7(hw)) 1035 return; 1036 1037 cmd_info->ringBufSize.ring1BufSizeType0 = hw->rxdata_buf_size; 1038 cmd_info->ringBufSize.ring1BufSizeType1 = 0; 1039 cmd_info->ringBufSize.ring2BufSizeType1 = hw->rxdata_buf_size; 1040 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1041 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 1042 } 1043 1044 /* 1045 * Configure device link speed and setup link. 1046 * Must be called after eth_vmxnet3_dev_init. Other wise it might fail 1047 * It returns 0 on success. 1048 */ 1049 static int 1050 vmxnet3_dev_start(struct rte_eth_dev *dev) 1051 { 1052 int ret; 1053 struct vmxnet3_hw *hw = dev->data->dev_private; 1054 uint16_t i; 1055 1056 PMD_INIT_FUNC_TRACE(); 1057 1058 /* Save stats before it is reset by CMD_ACTIVATE */ 1059 vmxnet3_hw_stats_save(hw); 1060 1061 /* configure MSI-X */ 1062 ret = vmxnet3_configure_msix(dev); 1063 if (ret < 0) { 1064 /* revert to lsc only */ 1065 hw->intr.num_intrs = 2; 1066 hw->intr.lsc_only = TRUE; 1067 } 1068 1069 ret = vmxnet3_setup_driver_shared(dev); 1070 if (ret != VMXNET3_SUCCESS) 1071 return ret; 1072 1073 /* Exchange shared data with device */ 1074 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 1075 VMXNET3_GET_ADDR_LO(hw->sharedPA)); 1076 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 1077 VMXNET3_GET_ADDR_HI(hw->sharedPA)); 1078 1079 /* Activate device by register write */ 1080 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV); 1081 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1082 1083 if (ret != 0) { 1084 PMD_INIT_LOG(ERR, "Device activation: UNSUCCESSFUL"); 1085 return -EINVAL; 1086 } 1087 1088 /* Check memregs restrictions first */ 1089 if (dev->data->nb_rx_queues <= VMXNET3_MAX_RX_QUEUES && 1090 dev->data->nb_tx_queues <= VMXNET3_MAX_TX_QUEUES) { 1091 ret = vmxnet3_dev_setup_memreg(dev); 1092 if (ret == 0) { 1093 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1094 VMXNET3_CMD_REGISTER_MEMREGS); 1095 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1096 if (ret != 0) 1097 PMD_INIT_LOG(DEBUG, 1098 "Failed in setup memory region cmd"); 1099 ret = 0; 1100 } else { 1101 PMD_INIT_LOG(DEBUG, "Failed to setup memory region"); 1102 } 1103 } else { 1104 PMD_INIT_LOG(WARNING, "Memregs can't init (rx: %d, tx: %d)", 1105 dev->data->nb_rx_queues, dev->data->nb_tx_queues); 1106 } 1107 1108 if (VMXNET3_VERSION_GE_4(hw) && 1109 dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 1110 /* Check for additional RSS */ 1111 ret = vmxnet3_v4_rss_configure(dev); 1112 if (ret != VMXNET3_SUCCESS) { 1113 PMD_INIT_LOG(ERR, "Failed to configure v4 RSS"); 1114 return ret; 1115 } 1116 } 1117 1118 /* 1119 * Load RX queues with blank mbufs and update next2fill index for device 1120 * Update RxMode of the device 1121 */ 1122 ret = vmxnet3_dev_rxtx_init(dev); 1123 if (ret != VMXNET3_SUCCESS) { 1124 PMD_INIT_LOG(ERR, "Device queue init: UNSUCCESSFUL"); 1125 return ret; 1126 } 1127 1128 vmxnet3_init_bufsize(hw); 1129 1130 hw->adapter_stopped = FALSE; 1131 1132 /* Setting proper Rx Mode and issue Rx Mode Update command */ 1133 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); 1134 1135 #ifndef RTE_EXEC_ENV_FREEBSD 1136 /* Setup interrupt callback */ 1137 rte_intr_callback_register(dev->intr_handle, 1138 vmxnet3_interrupt_handler, dev); 1139 1140 if (rte_intr_enable(dev->intr_handle) < 0) { 1141 PMD_INIT_LOG(ERR, "interrupt enable failed"); 1142 return -EIO; 1143 } 1144 1145 /* enable all intrs */ 1146 vmxnet3_enable_all_intrs(hw); 1147 #endif 1148 1149 vmxnet3_process_events(dev); 1150 1151 /* 1152 * Update link state from device since this won't be 1153 * done upon starting with lsc in use. This is done 1154 * only after enabling interrupts to avoid any race 1155 * where the link state could change without an 1156 * interrupt being fired. 1157 */ 1158 __vmxnet3_dev_link_update(dev, 0); 1159 1160 for (i = 0; i < dev->data->nb_rx_queues; i++) 1161 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 1162 for (i = 0; i < dev->data->nb_tx_queues; i++) 1163 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 1164 1165 return VMXNET3_SUCCESS; 1166 } 1167 1168 /* 1169 * Stop device: disable rx and tx functions to allow for reconfiguring. 1170 */ 1171 static int 1172 vmxnet3_dev_stop(struct rte_eth_dev *dev) 1173 { 1174 struct rte_eth_link link; 1175 struct vmxnet3_hw *hw = dev->data->dev_private; 1176 struct rte_intr_handle *intr_handle = dev->intr_handle; 1177 uint16_t i; 1178 int ret; 1179 1180 PMD_INIT_FUNC_TRACE(); 1181 1182 if (hw->adapter_stopped == 1) { 1183 PMD_INIT_LOG(DEBUG, "Device already stopped."); 1184 return 0; 1185 } 1186 1187 do { 1188 /* Unregister has lock to make sure there is no running cb. 1189 * This has to happen first since vmxnet3_interrupt_handler 1190 * reenables interrupts by calling vmxnet3_enable_intr 1191 */ 1192 ret = rte_intr_callback_unregister(intr_handle, 1193 vmxnet3_interrupt_handler, 1194 (void *)-1); 1195 } while (ret == -EAGAIN); 1196 1197 if (ret < 0) 1198 PMD_DRV_LOG(ERR, "Error attempting to unregister intr cb: %d", 1199 ret); 1200 1201 PMD_INIT_LOG(DEBUG, "Disabled %d intr callbacks", ret); 1202 1203 /* disable interrupts */ 1204 vmxnet3_disable_all_intrs(hw); 1205 1206 rte_intr_disable(intr_handle); 1207 1208 /* Clean datapath event and queue/vector mapping */ 1209 rte_intr_efd_disable(intr_handle); 1210 rte_intr_vec_list_free(intr_handle); 1211 1212 /* quiesce the device first */ 1213 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 1214 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0); 1215 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 0); 1216 1217 /* reset the device */ 1218 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 1219 PMD_INIT_LOG(DEBUG, "Device reset."); 1220 1221 vmxnet3_dev_clear_queues(dev); 1222 1223 /* Clear recorded link status */ 1224 memset(&link, 0, sizeof(link)); 1225 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 1226 link.link_speed = RTE_ETH_SPEED_NUM_10G; 1227 link.link_autoneg = RTE_ETH_LINK_FIXED; 1228 rte_eth_linkstatus_set(dev, &link); 1229 1230 hw->adapter_stopped = 1; 1231 dev->data->dev_started = 0; 1232 1233 for (i = 0; i < dev->data->nb_rx_queues; i++) 1234 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1235 for (i = 0; i < dev->data->nb_tx_queues; i++) 1236 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1237 1238 return 0; 1239 } 1240 1241 static void 1242 vmxnet3_free_queues(struct rte_eth_dev *dev) 1243 { 1244 int i; 1245 1246 PMD_INIT_FUNC_TRACE(); 1247 1248 for (i = 0; i < dev->data->nb_rx_queues; i++) 1249 vmxnet3_dev_rx_queue_release(dev, i); 1250 dev->data->nb_rx_queues = 0; 1251 1252 for (i = 0; i < dev->data->nb_tx_queues; i++) 1253 vmxnet3_dev_tx_queue_release(dev, i); 1254 dev->data->nb_tx_queues = 0; 1255 } 1256 1257 /* 1258 * Reset and stop device. 1259 */ 1260 static int 1261 vmxnet3_dev_close(struct rte_eth_dev *dev) 1262 { 1263 int ret; 1264 PMD_INIT_FUNC_TRACE(); 1265 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1266 return 0; 1267 1268 ret = vmxnet3_dev_stop(dev); 1269 vmxnet3_free_queues(dev); 1270 1271 return ret; 1272 } 1273 1274 static int 1275 vmxnet3_dev_reset(struct rte_eth_dev *dev) 1276 { 1277 int ret; 1278 1279 ret = eth_vmxnet3_dev_uninit(dev); 1280 if (ret) 1281 return ret; 1282 ret = eth_vmxnet3_dev_init(dev); 1283 return ret; 1284 } 1285 1286 static void 1287 vmxnet3_hw_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1288 struct UPT1_TxStats *res) 1289 { 1290 #define VMXNET3_UPDATE_TX_STAT(h, i, f, r) \ 1291 ((r)->f = (h)->tqd_start[(i)].stats.f + \ 1292 (h)->saved_tx_stats[(i)].f) 1293 1294 VMXNET3_UPDATE_TX_STAT(hw, q, ucastPktsTxOK, res); 1295 VMXNET3_UPDATE_TX_STAT(hw, q, mcastPktsTxOK, res); 1296 VMXNET3_UPDATE_TX_STAT(hw, q, bcastPktsTxOK, res); 1297 VMXNET3_UPDATE_TX_STAT(hw, q, ucastBytesTxOK, res); 1298 VMXNET3_UPDATE_TX_STAT(hw, q, mcastBytesTxOK, res); 1299 VMXNET3_UPDATE_TX_STAT(hw, q, bcastBytesTxOK, res); 1300 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxError, res); 1301 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxDiscard, res); 1302 1303 #undef VMXNET3_UPDATE_TX_STAT 1304 } 1305 1306 static void 1307 vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1308 struct UPT1_RxStats *res) 1309 { 1310 #define VMXNET3_UPDATE_RX_STAT(h, i, f, r) \ 1311 ((r)->f = (h)->rqd_start[(i)].stats.f + \ 1312 (h)->saved_rx_stats[(i)].f) 1313 1314 VMXNET3_UPDATE_RX_STAT(hw, q, ucastPktsRxOK, res); 1315 VMXNET3_UPDATE_RX_STAT(hw, q, mcastPktsRxOK, res); 1316 VMXNET3_UPDATE_RX_STAT(hw, q, bcastPktsRxOK, res); 1317 VMXNET3_UPDATE_RX_STAT(hw, q, ucastBytesRxOK, res); 1318 VMXNET3_UPDATE_RX_STAT(hw, q, mcastBytesRxOK, res); 1319 VMXNET3_UPDATE_RX_STAT(hw, q, bcastBytesRxOK, res); 1320 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res); 1321 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res); 1322 1323 #undef VMXNET3_UPDATE_RX_STAT 1324 } 1325 1326 static void 1327 vmxnet3_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1328 struct UPT1_TxStats *res) 1329 { 1330 vmxnet3_hw_tx_stats_get(hw, q, res); 1331 1332 #define VMXNET3_REDUCE_SNAPSHOT_TX_STAT(h, i, f, r) \ 1333 ((r)->f -= (h)->snapshot_tx_stats[(i)].f) 1334 1335 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastPktsTxOK, res); 1336 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastPktsTxOK, res); 1337 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastPktsTxOK, res); 1338 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastBytesTxOK, res); 1339 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastBytesTxOK, res); 1340 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastBytesTxOK, res); 1341 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxError, res); 1342 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxDiscard, res); 1343 1344 #undef VMXNET3_REDUCE_SNAPSHOT_TX_STAT 1345 } 1346 1347 static void 1348 vmxnet3_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1349 struct UPT1_RxStats *res) 1350 { 1351 vmxnet3_hw_rx_stats_get(hw, q, res); 1352 1353 #define VMXNET3_REDUCE_SNAPSHOT_RX_STAT(h, i, f, r) \ 1354 ((r)->f -= (h)->snapshot_rx_stats[(i)].f) 1355 1356 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastPktsRxOK, res); 1357 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastPktsRxOK, res); 1358 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastPktsRxOK, res); 1359 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastBytesRxOK, res); 1360 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastBytesRxOK, res); 1361 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastBytesRxOK, res); 1362 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxError, res); 1363 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxOutOfBuf, res); 1364 1365 #undef VMXNET3_REDUCE_SNAPSHOT_RX_STAT 1366 } 1367 1368 static void 1369 vmxnet3_hw_stats_save(struct vmxnet3_hw *hw) 1370 { 1371 unsigned int i; 1372 1373 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1374 1375 for (i = 0; i < hw->num_tx_queues; i++) 1376 vmxnet3_hw_tx_stats_get(hw, i, &hw->saved_tx_stats[i]); 1377 for (i = 0; i < hw->num_rx_queues; i++) 1378 vmxnet3_hw_rx_stats_get(hw, i, &hw->saved_rx_stats[i]); 1379 } 1380 1381 static int 1382 vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 1383 struct rte_eth_xstat_name *xstats_names, 1384 unsigned int n) 1385 { 1386 unsigned int i, t, count = 0; 1387 unsigned int nstats = 1388 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 1389 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 1390 1391 if (!xstats_names || n < nstats) 1392 return nstats; 1393 1394 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1395 if (!dev->data->rx_queues[i]) 1396 continue; 1397 1398 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 1399 snprintf(xstats_names[count].name, 1400 sizeof(xstats_names[count].name), 1401 "rx_q%u_%s", i, 1402 vmxnet3_rxq_stat_strings[t].name); 1403 count++; 1404 } 1405 } 1406 1407 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1408 if (!dev->data->tx_queues[i]) 1409 continue; 1410 1411 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 1412 snprintf(xstats_names[count].name, 1413 sizeof(xstats_names[count].name), 1414 "tx_q%u_%s", i, 1415 vmxnet3_txq_stat_strings[t].name); 1416 count++; 1417 } 1418 } 1419 1420 return count; 1421 } 1422 1423 static int 1424 vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1425 unsigned int n) 1426 { 1427 unsigned int i, t, count = 0; 1428 unsigned int nstats = 1429 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 1430 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 1431 1432 if (n < nstats) 1433 return nstats; 1434 1435 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1436 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i]; 1437 1438 if (rxq == NULL) 1439 continue; 1440 1441 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 1442 xstats[count].value = *(uint64_t *)(((char *)&rxq->stats) + 1443 vmxnet3_rxq_stat_strings[t].offset); 1444 xstats[count].id = count; 1445 count++; 1446 } 1447 } 1448 1449 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1450 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; 1451 1452 if (txq == NULL) 1453 continue; 1454 1455 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 1456 xstats[count].value = *(uint64_t *)(((char *)&txq->stats) + 1457 vmxnet3_txq_stat_strings[t].offset); 1458 xstats[count].id = count; 1459 count++; 1460 } 1461 } 1462 1463 return count; 1464 } 1465 1466 static int 1467 vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1468 { 1469 unsigned int i; 1470 struct vmxnet3_hw *hw = dev->data->dev_private; 1471 struct UPT1_TxStats txStats; 1472 struct UPT1_RxStats rxStats; 1473 1474 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1475 1476 for (i = 0; i < hw->num_tx_queues; i++) { 1477 vmxnet3_tx_stats_get(hw, i, &txStats); 1478 1479 stats->q_opackets[i] = txStats.ucastPktsTxOK + 1480 txStats.mcastPktsTxOK + 1481 txStats.bcastPktsTxOK; 1482 1483 stats->q_obytes[i] = txStats.ucastBytesTxOK + 1484 txStats.mcastBytesTxOK + 1485 txStats.bcastBytesTxOK; 1486 1487 stats->opackets += stats->q_opackets[i]; 1488 stats->obytes += stats->q_obytes[i]; 1489 stats->oerrors += txStats.pktsTxError + txStats.pktsTxDiscard; 1490 } 1491 1492 for (i = 0; i < hw->num_rx_queues; i++) { 1493 vmxnet3_rx_stats_get(hw, i, &rxStats); 1494 1495 stats->q_ipackets[i] = rxStats.ucastPktsRxOK + 1496 rxStats.mcastPktsRxOK + 1497 rxStats.bcastPktsRxOK; 1498 1499 stats->q_ibytes[i] = rxStats.ucastBytesRxOK + 1500 rxStats.mcastBytesRxOK + 1501 rxStats.bcastBytesRxOK; 1502 1503 stats->ipackets += stats->q_ipackets[i]; 1504 stats->ibytes += stats->q_ibytes[i]; 1505 1506 stats->q_errors[i] = rxStats.pktsRxError; 1507 stats->ierrors += rxStats.pktsRxError; 1508 stats->imissed += rxStats.pktsRxOutOfBuf; 1509 } 1510 1511 return 0; 1512 } 1513 1514 static int 1515 vmxnet3_dev_stats_reset(struct rte_eth_dev *dev) 1516 { 1517 unsigned int i; 1518 struct vmxnet3_hw *hw = dev->data->dev_private; 1519 struct UPT1_TxStats txStats = {0}; 1520 struct UPT1_RxStats rxStats = {0}; 1521 1522 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1523 1524 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); 1525 1526 for (i = 0; i < hw->num_tx_queues; i++) { 1527 vmxnet3_hw_tx_stats_get(hw, i, &txStats); 1528 memcpy(&hw->snapshot_tx_stats[i], &txStats, 1529 sizeof(hw->snapshot_tx_stats[0])); 1530 } 1531 for (i = 0; i < hw->num_rx_queues; i++) { 1532 vmxnet3_hw_rx_stats_get(hw, i, &rxStats); 1533 memcpy(&hw->snapshot_rx_stats[i], &rxStats, 1534 sizeof(hw->snapshot_rx_stats[0])); 1535 } 1536 1537 return 0; 1538 } 1539 1540 static int 1541 vmxnet3_dev_info_get(struct rte_eth_dev *dev, 1542 struct rte_eth_dev_info *dev_info) 1543 { 1544 struct vmxnet3_hw *hw = dev->data->dev_private; 1545 int queues = 0; 1546 1547 if (VMXNET3_VERSION_GE_6(hw)) { 1548 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1549 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 1550 queues = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1551 1552 if (queues > 0) { 1553 dev_info->max_rx_queues = 1554 RTE_MIN(VMXNET3_EXT_MAX_RX_QUEUES, ((queues >> 8) & 0xff)); 1555 dev_info->max_tx_queues = 1556 RTE_MIN(VMXNET3_EXT_MAX_TX_QUEUES, (queues & 0xff)); 1557 } else { 1558 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; 1559 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; 1560 } 1561 } else { 1562 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; 1563 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; 1564 } 1565 1566 dev_info->min_rx_bufsize = 1518 + RTE_PKTMBUF_HEADROOM; 1567 dev_info->max_rx_pktlen = 16384; /* includes CRC, cf MAXFRS register */ 1568 dev_info->min_mtu = VMXNET3_MIN_MTU; 1569 dev_info->max_mtu = VMXNET3_MAX_MTU; 1570 dev_info->speed_capa = RTE_ETH_LINK_SPEED_10G; 1571 dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS; 1572 1573 dev_info->flow_type_rss_offloads = VMXNET3_RSS_OFFLOAD_ALL; 1574 1575 if (VMXNET3_VERSION_GE_4(hw)) { 1576 dev_info->flow_type_rss_offloads |= VMXNET3_V4_RSS_MASK; 1577 } 1578 1579 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 1580 .nb_max = VMXNET3_RX_RING_MAX_SIZE, 1581 .nb_min = VMXNET3_DEF_RX_RING_SIZE, 1582 .nb_align = 1, 1583 }; 1584 1585 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 1586 .nb_max = VMXNET3_TX_RING_MAX_SIZE, 1587 .nb_min = VMXNET3_DEF_TX_RING_SIZE, 1588 .nb_align = 1, 1589 .nb_seg_max = VMXNET3_TX_MAX_SEG, 1590 .nb_mtu_seg_max = VMXNET3_MAX_TXD_PER_PKT, 1591 }; 1592 1593 dev_info->rx_offload_capa = VMXNET3_RX_OFFLOAD_CAP; 1594 dev_info->rx_queue_offload_capa = 0; 1595 dev_info->tx_offload_capa = VMXNET3_TX_OFFLOAD_CAP; 1596 dev_info->tx_queue_offload_capa = 0; 1597 if (hw->rss_conf == NULL) { 1598 /* RSS not configured */ 1599 dev_info->reta_size = 0; 1600 } else { 1601 dev_info->reta_size = hw->rss_conf->indTableSize; 1602 } 1603 return 0; 1604 } 1605 1606 static int 1607 vmxnet3_hw_ver_get(struct rte_eth_dev *dev, 1608 char *fw_version, size_t fw_size) 1609 { 1610 int ret; 1611 struct vmxnet3_hw *hw = dev->data->dev_private; 1612 1613 ret = snprintf(fw_version, fw_size, "v%d", hw->version); 1614 1615 ret += 1; /* add the size of '\0' */ 1616 if (fw_size < (uint32_t)ret) 1617 return ret; 1618 else 1619 return 0; 1620 } 1621 1622 static const uint32_t * 1623 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev, 1624 size_t *no_of_elements) 1625 { 1626 static const uint32_t ptypes[] = { 1627 RTE_PTYPE_L3_IPV4_EXT, 1628 RTE_PTYPE_L3_IPV4, 1629 }; 1630 1631 if (dev->rx_pkt_burst == vmxnet3_recv_pkts) { 1632 *no_of_elements = RTE_DIM(ptypes); 1633 return ptypes; 1634 } 1635 return NULL; 1636 } 1637 1638 static int 1639 vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) 1640 { 1641 struct vmxnet3_hw *hw = dev->data->dev_private; 1642 1643 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)(hw->perm_addr)); 1644 vmxnet3_write_mac(hw, mac_addr->addr_bytes); 1645 return 0; 1646 } 1647 1648 static int 1649 vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1650 { 1651 struct vmxnet3_hw *hw = dev->data->dev_private; 1652 uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 4; 1653 1654 if (mtu < VMXNET3_MIN_MTU) 1655 return -EINVAL; 1656 1657 if (VMXNET3_VERSION_GE_6(hw)) { 1658 if (mtu > VMXNET3_V6_MAX_MTU) 1659 return -EINVAL; 1660 } else { 1661 if (mtu > VMXNET3_MAX_MTU) { 1662 PMD_DRV_LOG(ERR, "MTU %d too large in device version v%d", 1663 mtu, hw->version); 1664 return -EINVAL; 1665 } 1666 } 1667 1668 dev->data->mtu = mtu; 1669 /* update max frame size */ 1670 dev->data->dev_conf.rxmode.mtu = frame_size; 1671 1672 if (dev->data->dev_started == 0) 1673 return 0; 1674 1675 /* changing mtu for vmxnet3 pmd does not require a restart 1676 * as it does not need to repopulate the rx rings to support 1677 * different mtu size. We stop and restart the device here 1678 * just to pass the mtu info to the backend. 1679 */ 1680 vmxnet3_dev_stop(dev); 1681 vmxnet3_dev_start(dev); 1682 1683 return 0; 1684 } 1685 1686 /* return 0 means link status changed, -1 means not changed */ 1687 static int 1688 __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 1689 __rte_unused int wait_to_complete) 1690 { 1691 struct vmxnet3_hw *hw = dev->data->dev_private; 1692 struct rte_eth_link link; 1693 uint32_t ret; 1694 1695 memset(&link, 0, sizeof(link)); 1696 1697 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 1698 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1699 1700 if (ret & 0x1) 1701 link.link_status = RTE_ETH_LINK_UP; 1702 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 1703 link.link_speed = RTE_ETH_SPEED_NUM_10G; 1704 link.link_autoneg = RTE_ETH_LINK_FIXED; 1705 1706 return rte_eth_linkstatus_set(dev, &link); 1707 } 1708 1709 static int 1710 vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 1711 { 1712 /* Link status doesn't change for stopped dev */ 1713 if (dev->data->dev_started == 0) 1714 return -1; 1715 1716 return __vmxnet3_dev_link_update(dev, wait_to_complete); 1717 } 1718 1719 /* Updating rxmode through Vmxnet3_DriverShared structure in adapter */ 1720 static void 1721 vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) 1722 { 1723 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1724 1725 if (set) 1726 rxConf->rxMode = rxConf->rxMode | feature; 1727 else 1728 rxConf->rxMode = rxConf->rxMode & (~feature); 1729 1730 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE); 1731 } 1732 1733 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1734 static int 1735 vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev) 1736 { 1737 struct vmxnet3_hw *hw = dev->data->dev_private; 1738 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1739 1740 memset(vf_table, 0, VMXNET3_VFT_TABLE_SIZE); 1741 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 1); 1742 1743 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1744 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1745 1746 return 0; 1747 } 1748 1749 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1750 static int 1751 vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev) 1752 { 1753 struct vmxnet3_hw *hw = dev->data->dev_private; 1754 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1755 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 1756 1757 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) 1758 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1759 else 1760 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1761 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0); 1762 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1763 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1764 1765 return 0; 1766 } 1767 1768 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1769 static int 1770 vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev) 1771 { 1772 struct vmxnet3_hw *hw = dev->data->dev_private; 1773 1774 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1); 1775 1776 return 0; 1777 } 1778 1779 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1780 static int 1781 vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev) 1782 { 1783 struct vmxnet3_hw *hw = dev->data->dev_private; 1784 1785 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 0); 1786 1787 return 0; 1788 } 1789 1790 /* Enable/disable filter on vlan */ 1791 static int 1792 vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vid, int on) 1793 { 1794 struct vmxnet3_hw *hw = dev->data->dev_private; 1795 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1796 uint32_t *vf_table = rxConf->vfTable; 1797 1798 /* save state for restore */ 1799 if (on) 1800 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1801 else 1802 VMXNET3_CLEAR_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1803 1804 /* don't change active filter if in promiscuous mode */ 1805 if (rxConf->rxMode & VMXNET3_RXM_PROMISC) 1806 return 0; 1807 1808 /* set in hardware */ 1809 if (on) 1810 VMXNET3_SET_VFTABLE_ENTRY(vf_table, vid); 1811 else 1812 VMXNET3_CLEAR_VFTABLE_ENTRY(vf_table, vid); 1813 1814 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1815 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1816 return 0; 1817 } 1818 1819 static int 1820 vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1821 { 1822 struct vmxnet3_hw *hw = dev->data->dev_private; 1823 Vmxnet3_DSDevRead *devRead = &hw->shared->devRead; 1824 uint32_t *vf_table = devRead->rxFilterConf.vfTable; 1825 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 1826 1827 if (mask & RTE_ETH_VLAN_STRIP_MASK) { 1828 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 1829 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 1830 else 1831 devRead->misc.uptFeatures &= ~UPT1_F_RXVLAN; 1832 1833 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1834 VMXNET3_CMD_UPDATE_FEATURE); 1835 } 1836 1837 if (mask & RTE_ETH_VLAN_FILTER_MASK) { 1838 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) 1839 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1840 else 1841 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1842 1843 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1844 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1845 } 1846 1847 return 0; 1848 } 1849 1850 static void 1851 vmxnet3_process_events(struct rte_eth_dev *dev) 1852 { 1853 struct vmxnet3_hw *hw = dev->data->dev_private; 1854 uint32_t events = hw->shared->ecr; 1855 int i; 1856 1857 if (!events) 1858 return; 1859 1860 /* 1861 * ECR bits when written with 1b are cleared. Hence write 1862 * events back to ECR so that the bits which were set will be reset. 1863 */ 1864 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events); 1865 1866 /* Check if link state has changed */ 1867 if (events & VMXNET3_ECR_LINK) { 1868 PMD_DRV_LOG(DEBUG, "Process events: VMXNET3_ECR_LINK event"); 1869 if (vmxnet3_dev_link_update(dev, 0) == 0) 1870 rte_eth_dev_callback_process(dev, 1871 RTE_ETH_EVENT_INTR_LSC, 1872 NULL); 1873 } 1874 1875 /* Check if there is an error on xmit/recv queues */ 1876 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 1877 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1878 VMXNET3_CMD_GET_QUEUE_STATUS); 1879 1880 PMD_DRV_LOG(ERR, "queue error event 0x%x for " 1881 RTE_ETHER_ADDR_PRT_FMT, events, 1882 hw->perm_addr[0], hw->perm_addr[1], 1883 hw->perm_addr[2], hw->perm_addr[3], 1884 hw->perm_addr[4], hw->perm_addr[5]); 1885 1886 for (i = 0; i < hw->num_tx_queues; i++) { 1887 if (hw->tqd_start[i].status.stopped) 1888 PMD_DRV_LOG(ERR, "tq %d error 0x%x", 1889 i, hw->tqd_start[i].status.error); 1890 } 1891 for (i = 0; i < hw->num_rx_queues; i++) { 1892 if (hw->rqd_start[i].status.stopped) 1893 PMD_DRV_LOG(ERR, "rq %d error 0x%x", 1894 i, hw->rqd_start[i].status.error); 1895 } 1896 1897 /* Have to reset the device */ 1898 /* Notify the application so that it can reset the device */ 1899 rte_eth_dev_callback_process(dev, 1900 RTE_ETH_EVENT_INTR_RESET, 1901 NULL); 1902 } 1903 1904 if (events & VMXNET3_ECR_DIC) 1905 PMD_DRV_LOG(DEBUG, "Device implementation change event."); 1906 1907 if (events & VMXNET3_ECR_DEBUG) 1908 PMD_DRV_LOG(DEBUG, "Debug event generated by device."); 1909 } 1910 1911 static void 1912 vmxnet3_interrupt_handler(void *param) 1913 { 1914 struct rte_eth_dev *dev = param; 1915 struct vmxnet3_hw *hw = dev->data->dev_private; 1916 uint32_t events; 1917 uint8 *eventIntrIdx; 1918 uint32 *intrCtrl; 1919 1920 PMD_INIT_FUNC_TRACE(); 1921 1922 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 1923 vmxnet3_disable_intr(hw, *eventIntrIdx); 1924 1925 events = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_ECR); 1926 if (events == 0) 1927 goto done; 1928 1929 PMD_DRV_LOG(DEBUG, "Reading events: 0x%X", events); 1930 vmxnet3_process_events(dev); 1931 done: 1932 vmxnet3_enable_intr(hw, *eventIntrIdx); 1933 } 1934 1935 static int 1936 vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1937 { 1938 #ifndef RTE_EXEC_ENV_FREEBSD 1939 struct vmxnet3_hw *hw = dev->data->dev_private; 1940 1941 vmxnet3_enable_intr(hw, 1942 rte_intr_vec_list_index_get(dev->intr_handle, 1943 queue_id)); 1944 #endif 1945 1946 return 0; 1947 } 1948 1949 static int 1950 vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1951 { 1952 struct vmxnet3_hw *hw = dev->data->dev_private; 1953 1954 vmxnet3_disable_intr(hw, 1955 rte_intr_vec_list_index_get(dev->intr_handle, queue_id)); 1956 1957 return 0; 1958 } 1959 1960 RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd); 1961 RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map); 1962 RTE_PMD_REGISTER_KMOD_DEP(net_vmxnet3, "* igb_uio | uio_pci_generic | vfio-pci"); 1963 RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_init, init, NOTICE); 1964 RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_driver, driver, NOTICE); 1965 1966 static int 1967 vmxnet3_rss_reta_update(struct rte_eth_dev *dev, 1968 struct rte_eth_rss_reta_entry64 *reta_conf, 1969 uint16_t reta_size) 1970 { 1971 int i, idx, shift; 1972 struct vmxnet3_hw *hw = dev->data->dev_private; 1973 struct VMXNET3_RSSConf *dev_rss_conf = hw->rss_conf; 1974 1975 if (reta_size != dev_rss_conf->indTableSize) { 1976 PMD_DRV_LOG(ERR, 1977 "The size of hash lookup table configured (%d) doesn't match " 1978 "the supported number (%d)", 1979 reta_size, dev_rss_conf->indTableSize); 1980 return -EINVAL; 1981 } 1982 1983 for (i = 0; i < reta_size; i++) { 1984 idx = i / RTE_ETH_RETA_GROUP_SIZE; 1985 shift = i % RTE_ETH_RETA_GROUP_SIZE; 1986 if (reta_conf[idx].mask & RTE_BIT64(shift)) 1987 dev_rss_conf->indTable[i] = (uint8_t)reta_conf[idx].reta[shift]; 1988 } 1989 1990 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1991 VMXNET3_CMD_UPDATE_RSSIDT); 1992 1993 return 0; 1994 } 1995 1996 static int 1997 vmxnet3_rss_reta_query(struct rte_eth_dev *dev, 1998 struct rte_eth_rss_reta_entry64 *reta_conf, 1999 uint16_t reta_size) 2000 { 2001 int i, idx, shift; 2002 struct vmxnet3_hw *hw = dev->data->dev_private; 2003 struct VMXNET3_RSSConf *dev_rss_conf = hw->rss_conf; 2004 2005 if (reta_size != dev_rss_conf->indTableSize) { 2006 PMD_DRV_LOG(ERR, 2007 "Size of requested hash lookup table (%d) doesn't " 2008 "match the configured size (%d)", 2009 reta_size, dev_rss_conf->indTableSize); 2010 return -EINVAL; 2011 } 2012 2013 for (i = 0; i < reta_size; i++) { 2014 idx = i / RTE_ETH_RETA_GROUP_SIZE; 2015 shift = i % RTE_ETH_RETA_GROUP_SIZE; 2016 if (reta_conf[idx].mask & RTE_BIT64(shift)) 2017 reta_conf[idx].reta[shift] = dev_rss_conf->indTable[i]; 2018 } 2019 2020 return 0; 2021 } 2022