1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdio.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <unistd.h> 12 #include <stdarg.h> 13 #include <fcntl.h> 14 #include <inttypes.h> 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 19 #include <rte_interrupts.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_pci.h> 23 #include <bus_pci_driver.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_memory.h> 26 #include <rte_memzone.h> 27 #include <rte_eal.h> 28 #include <rte_alarm.h> 29 #include <rte_ether.h> 30 #include <ethdev_driver.h> 31 #include <ethdev_pci.h> 32 #include <rte_string_fns.h> 33 #include <rte_malloc.h> 34 #include <dev_driver.h> 35 36 #include "base/vmxnet3_defs.h" 37 38 #include "vmxnet3_ring.h" 39 #include "vmxnet3_logs.h" 40 #include "vmxnet3_ethdev.h" 41 42 #define VMXNET3_TX_MAX_SEG UINT8_MAX 43 44 #define VMXNET3_TX_OFFLOAD_CAP \ 45 (RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ 46 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \ 47 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \ 48 RTE_ETH_TX_OFFLOAD_TCP_TSO | \ 49 RTE_ETH_TX_OFFLOAD_MULTI_SEGS) 50 51 #define VMXNET3_RX_OFFLOAD_CAP \ 52 (RTE_ETH_RX_OFFLOAD_VLAN_STRIP | \ 53 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | \ 54 RTE_ETH_RX_OFFLOAD_SCATTER | \ 55 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | \ 56 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | \ 57 RTE_ETH_RX_OFFLOAD_TCP_LRO | \ 58 RTE_ETH_RX_OFFLOAD_RSS_HASH) 59 60 int vmxnet3_segs_dynfield_offset = -1; 61 62 static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev); 63 static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev); 64 static int vmxnet3_dev_configure(struct rte_eth_dev *dev); 65 static int vmxnet3_dev_start(struct rte_eth_dev *dev); 66 static int vmxnet3_dev_stop(struct rte_eth_dev *dev); 67 static int vmxnet3_dev_close(struct rte_eth_dev *dev); 68 static int vmxnet3_dev_reset(struct rte_eth_dev *dev); 69 static void vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set); 70 static int vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev); 71 static int vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev); 72 static int vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev); 73 static int vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev); 74 static int __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 75 int wait_to_complete); 76 static int vmxnet3_dev_link_update(struct rte_eth_dev *dev, 77 int wait_to_complete); 78 static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw); 79 static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev, 80 struct rte_eth_stats *stats); 81 static int vmxnet3_dev_stats_reset(struct rte_eth_dev *dev); 82 static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 83 struct rte_eth_xstat_name *xstats, 84 unsigned int n); 85 static int vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, 86 struct rte_eth_xstat *xstats, unsigned int n); 87 static int vmxnet3_dev_info_get(struct rte_eth_dev *dev, 88 struct rte_eth_dev_info *dev_info); 89 static int vmxnet3_hw_ver_get(struct rte_eth_dev *dev, 90 char *fw_version, size_t fw_size); 91 static const uint32_t * 92 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev, 93 size_t *no_of_elements); 94 static int vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 95 static int vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, 96 uint16_t vid, int on); 97 static int vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 98 static int vmxnet3_mac_addr_set(struct rte_eth_dev *dev, 99 struct rte_ether_addr *mac_addr); 100 static void vmxnet3_process_events(struct rte_eth_dev *dev); 101 static void vmxnet3_interrupt_handler(void *param); 102 static int 103 vmxnet3_rss_reta_update(struct rte_eth_dev *dev, 104 struct rte_eth_rss_reta_entry64 *reta_conf, 105 uint16_t reta_size); 106 static int 107 vmxnet3_rss_reta_query(struct rte_eth_dev *dev, 108 struct rte_eth_rss_reta_entry64 *reta_conf, 109 uint16_t reta_size); 110 111 static int vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 112 uint16_t queue_id); 113 static int vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 114 uint16_t queue_id); 115 116 /* 117 * The set of PCI devices this driver supports 118 */ 119 #define VMWARE_PCI_VENDOR_ID 0x15AD 120 #define VMWARE_DEV_ID_VMXNET3 0x07B0 121 static const struct rte_pci_id pci_id_vmxnet3_map[] = { 122 { RTE_PCI_DEVICE(VMWARE_PCI_VENDOR_ID, VMWARE_DEV_ID_VMXNET3) }, 123 { .vendor_id = 0, /* sentinel */ }, 124 }; 125 126 static const struct eth_dev_ops vmxnet3_eth_dev_ops = { 127 .dev_configure = vmxnet3_dev_configure, 128 .dev_start = vmxnet3_dev_start, 129 .dev_stop = vmxnet3_dev_stop, 130 .dev_close = vmxnet3_dev_close, 131 .dev_reset = vmxnet3_dev_reset, 132 .link_update = vmxnet3_dev_link_update, 133 .promiscuous_enable = vmxnet3_dev_promiscuous_enable, 134 .promiscuous_disable = vmxnet3_dev_promiscuous_disable, 135 .allmulticast_enable = vmxnet3_dev_allmulticast_enable, 136 .allmulticast_disable = vmxnet3_dev_allmulticast_disable, 137 .mac_addr_set = vmxnet3_mac_addr_set, 138 .mtu_set = vmxnet3_dev_mtu_set, 139 .stats_get = vmxnet3_dev_stats_get, 140 .stats_reset = vmxnet3_dev_stats_reset, 141 .xstats_get = vmxnet3_dev_xstats_get, 142 .xstats_get_names = vmxnet3_dev_xstats_get_names, 143 .dev_infos_get = vmxnet3_dev_info_get, 144 .fw_version_get = vmxnet3_hw_ver_get, 145 .dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get, 146 .vlan_filter_set = vmxnet3_dev_vlan_filter_set, 147 .vlan_offload_set = vmxnet3_dev_vlan_offload_set, 148 .rx_queue_setup = vmxnet3_dev_rx_queue_setup, 149 .rx_queue_release = vmxnet3_dev_rx_queue_release, 150 .rx_queue_intr_enable = vmxnet3_dev_rx_queue_intr_enable, 151 .rx_queue_intr_disable = vmxnet3_dev_rx_queue_intr_disable, 152 .tx_queue_setup = vmxnet3_dev_tx_queue_setup, 153 .tx_queue_release = vmxnet3_dev_tx_queue_release, 154 .reta_update = vmxnet3_rss_reta_update, 155 .reta_query = vmxnet3_rss_reta_query, 156 }; 157 158 struct vmxnet3_xstats_name_off { 159 char name[RTE_ETH_XSTATS_NAME_SIZE]; 160 unsigned int offset; 161 }; 162 163 /* tx_qX_ is prepended to the name string here */ 164 static const struct vmxnet3_xstats_name_off vmxnet3_txq_stat_strings[] = { 165 {"drop_total", offsetof(struct vmxnet3_txq_stats, drop_total)}, 166 {"drop_too_many_segs", offsetof(struct vmxnet3_txq_stats, drop_too_many_segs)}, 167 {"drop_tso", offsetof(struct vmxnet3_txq_stats, drop_tso)}, 168 {"tx_ring_full", offsetof(struct vmxnet3_txq_stats, tx_ring_full)}, 169 }; 170 171 /* rx_qX_ is prepended to the name string here */ 172 static const struct vmxnet3_xstats_name_off vmxnet3_rxq_stat_strings[] = { 173 {"drop_total", offsetof(struct vmxnet3_rxq_stats, drop_total)}, 174 {"drop_err", offsetof(struct vmxnet3_rxq_stats, drop_err)}, 175 {"drop_fcs", offsetof(struct vmxnet3_rxq_stats, drop_fcs)}, 176 {"rx_buf_alloc_failure", offsetof(struct vmxnet3_rxq_stats, rx_buf_alloc_failure)}, 177 }; 178 179 static const struct rte_memzone * 180 gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size, 181 const char *post_string, int socket_id, 182 uint16_t align, bool reuse) 183 { 184 char z_name[RTE_MEMZONE_NAMESIZE]; 185 const struct rte_memzone *mz; 186 187 snprintf(z_name, sizeof(z_name), "eth_p%d_%s", 188 dev->data->port_id, post_string); 189 190 mz = rte_memzone_lookup(z_name); 191 if (!reuse) { 192 if (mz) 193 rte_memzone_free(mz); 194 return rte_memzone_reserve_aligned(z_name, size, socket_id, 195 RTE_MEMZONE_IOVA_CONTIG, align); 196 } 197 198 if (mz) 199 return mz; 200 201 return rte_memzone_reserve_aligned(z_name, size, socket_id, 202 RTE_MEMZONE_IOVA_CONTIG, align); 203 } 204 205 /* 206 * Enable the given interrupt 207 */ 208 static void 209 vmxnet3_enable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx) 210 { 211 PMD_INIT_FUNC_TRACE(); 212 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 0); 213 } 214 215 /* 216 * Disable the given interrupt 217 */ 218 static void 219 vmxnet3_disable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx) 220 { 221 PMD_INIT_FUNC_TRACE(); 222 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 1); 223 } 224 225 /* 226 * Simple helper to get intrCtrl and eventIntrIdx based on config and hw version 227 */ 228 static void 229 vmxnet3_get_intr_ctrl_ev(struct vmxnet3_hw *hw, 230 uint8 **out_eventIntrIdx, 231 uint32 **out_intrCtrl) 232 { 233 234 if (VMXNET3_VERSION_GE_6(hw) && hw->queuesExtEnabled) { 235 *out_eventIntrIdx = &hw->shared->devReadExt.intrConfExt.eventIntrIdx; 236 *out_intrCtrl = &hw->shared->devReadExt.intrConfExt.intrCtrl; 237 } else { 238 *out_eventIntrIdx = &hw->shared->devRead.intrConf.eventIntrIdx; 239 *out_intrCtrl = &hw->shared->devRead.intrConf.intrCtrl; 240 } 241 } 242 243 /* 244 * Disable all intrs used by the device 245 */ 246 static void 247 vmxnet3_disable_all_intrs(struct vmxnet3_hw *hw) 248 { 249 int i; 250 uint8 *eventIntrIdx; 251 uint32 *intrCtrl; 252 253 PMD_INIT_FUNC_TRACE(); 254 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 255 256 *intrCtrl |= rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 257 258 for (i = 0; i < hw->intr.num_intrs; i++) 259 vmxnet3_disable_intr(hw, i); 260 } 261 262 #ifndef RTE_EXEC_ENV_FREEBSD 263 /* 264 * Enable all intrs used by the device 265 */ 266 static void 267 vmxnet3_enable_all_intrs(struct vmxnet3_hw *hw) 268 { 269 uint8 *eventIntrIdx; 270 uint32 *intrCtrl; 271 272 PMD_INIT_FUNC_TRACE(); 273 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 274 275 *intrCtrl &= rte_cpu_to_le_32(~VMXNET3_IC_DISABLE_ALL); 276 277 if (hw->intr.lsc_only) { 278 vmxnet3_enable_intr(hw, *eventIntrIdx); 279 } else { 280 int i; 281 282 for (i = 0; i < hw->intr.num_intrs; i++) 283 vmxnet3_enable_intr(hw, i); 284 } 285 } 286 #endif 287 288 /* 289 * Gets tx data ring descriptor size. 290 */ 291 static uint16_t 292 eth_vmxnet3_txdata_get(struct vmxnet3_hw *hw) 293 { 294 uint16 txdata_desc_size; 295 296 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 297 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 298 txdata_desc_size = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 299 300 return (txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE || 301 txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE || 302 txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK) ? 303 sizeof(struct Vmxnet3_TxDataDesc) : txdata_desc_size; 304 } 305 306 static int 307 eth_vmxnet3_setup_capabilities(struct vmxnet3_hw *hw, 308 struct rte_eth_dev *eth_dev) 309 { 310 uint32_t dcr, ptcr, value; 311 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 312 313 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 314 VMXNET3_CMD_GET_MAX_CAPABILITIES); 315 value = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 316 hw->max_capabilities[0] = value; 317 dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_DCR); 318 hw->DCR_capabilities[0] = dcr; 319 hw->used_DCR_capabilities[0] = 0; 320 ptcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_PTCR); 321 hw->PTCR_capabilities[0] = ptcr; 322 hw->used_PTCR_capabilities[0] = 0; 323 324 if (hw->uptv2_enabled && !(ptcr & (1 << VMXNET3_DCR_ERROR))) { 325 PMD_DRV_LOG(NOTICE, "UPTv2 enabled"); 326 hw->used_PTCR_capabilities[0] = ptcr; 327 } else { 328 /* Use all DCR capabilities, but disable large bar */ 329 hw->used_DCR_capabilities[0] = dcr & 330 (~(1UL << VMXNET3_CAP_LARGE_BAR)); 331 PMD_DRV_LOG(NOTICE, "UPTv2 disabled"); 332 } 333 if (hw->DCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 334 hw->PTCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 335 if (hw->uptv2_enabled) { 336 hw->used_PTCR_capabilities[0] |= 337 (1UL << VMXNET3_CAP_OOORX_COMP); 338 } 339 } 340 if (hw->used_PTCR_capabilities[0]) { 341 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR, 342 hw->used_PTCR_capabilities[0]); 343 } else if (hw->used_DCR_capabilities[0]) { 344 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR, 345 hw->used_DCR_capabilities[0]); 346 } 347 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 348 dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 349 hw->used_DCR_capabilities[0] = dcr; 350 PMD_DRV_LOG(DEBUG, "Dev " PCI_PRI_FMT ", vmxnet3 v%d, UPT enabled: %s, " 351 "DCR0=0x%08x, used DCR=0x%08x, " 352 "PTCR=0x%08x, used PTCR=0x%08x", 353 pci_dev->addr.domain, pci_dev->addr.bus, 354 pci_dev->addr.devid, pci_dev->addr.function, hw->version, 355 hw->uptv2_enabled ? "true" : "false", 356 hw->DCR_capabilities[0], hw->used_DCR_capabilities[0], 357 hw->PTCR_capabilities[0], hw->used_PTCR_capabilities[0]); 358 return 0; 359 } 360 361 /* 362 * It returns 0 on success. 363 */ 364 static int 365 eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) 366 { 367 struct rte_pci_device *pci_dev; 368 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 369 uint32_t mac_hi, mac_lo, ver; 370 struct rte_eth_link link; 371 static const struct rte_mbuf_dynfield vmxnet3_segs_dynfield_desc = { 372 .name = VMXNET3_SEGS_DYNFIELD_NAME, 373 .size = sizeof(vmxnet3_segs_dynfield_t), 374 .align = alignof(vmxnet3_segs_dynfield_t), 375 }; 376 377 PMD_INIT_FUNC_TRACE(); 378 379 eth_dev->dev_ops = &vmxnet3_eth_dev_ops; 380 eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts; 381 eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts; 382 eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts; 383 eth_dev->rx_queue_count = vmxnet3_dev_rx_queue_count; 384 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 385 386 /* extra mbuf field is required to guess MSS */ 387 vmxnet3_segs_dynfield_offset = 388 rte_mbuf_dynfield_register(&vmxnet3_segs_dynfield_desc); 389 if (vmxnet3_segs_dynfield_offset < 0) { 390 PMD_INIT_LOG(ERR, "Cannot register mbuf field."); 391 return -rte_errno; 392 } 393 394 /* 395 * for secondary processes, we don't initialize any further as primary 396 * has already done this work. 397 */ 398 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 399 return 0; 400 401 rte_eth_copy_pci_info(eth_dev, pci_dev); 402 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 403 404 /* Vendor and Device ID need to be set before init of shared code */ 405 hw->device_id = pci_dev->id.device_id; 406 hw->vendor_id = pci_dev->id.vendor_id; 407 hw->hw_addr0 = (void *)pci_dev->mem_resource[0].addr; 408 hw->hw_addr1 = (void *)pci_dev->mem_resource[1].addr; 409 410 hw->num_rx_queues = 1; 411 hw->num_tx_queues = 1; 412 hw->bufs_per_pkt = 1; 413 414 /* Check h/w version compatibility with driver. */ 415 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_VRRS); 416 417 if (ver & (1 << VMXNET3_REV_7)) { 418 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 419 1 << VMXNET3_REV_7); 420 hw->version = VMXNET3_REV_7 + 1; 421 } else if (ver & (1 << VMXNET3_REV_6)) { 422 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 423 1 << VMXNET3_REV_6); 424 hw->version = VMXNET3_REV_6 + 1; 425 } else if (ver & (1 << VMXNET3_REV_5)) { 426 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 427 1 << VMXNET3_REV_5); 428 hw->version = VMXNET3_REV_5 + 1; 429 } else if (ver & (1 << VMXNET3_REV_4)) { 430 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 431 1 << VMXNET3_REV_4); 432 hw->version = VMXNET3_REV_4 + 1; 433 } else if (ver & (1 << VMXNET3_REV_3)) { 434 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 435 1 << VMXNET3_REV_3); 436 hw->version = VMXNET3_REV_3 + 1; 437 } else if (ver & (1 << VMXNET3_REV_2)) { 438 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 439 1 << VMXNET3_REV_2); 440 hw->version = VMXNET3_REV_2 + 1; 441 } else if (ver & (1 << VMXNET3_REV_1)) { 442 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 443 1 << VMXNET3_REV_1); 444 hw->version = VMXNET3_REV_1 + 1; 445 } else { 446 PMD_INIT_LOG(ERR, "Incompatible hardware version: %d", ver); 447 return -EIO; 448 } 449 450 PMD_INIT_LOG(INFO, "Using device v%d", hw->version); 451 452 /* Check UPT version compatibility with driver. */ 453 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_UVRS); 454 PMD_INIT_LOG(DEBUG, "UPT hardware version : %d", ver); 455 if (ver & 0x1) 456 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_UVRS, 1); 457 else { 458 PMD_INIT_LOG(ERR, "Incompatible UPT version."); 459 return -EIO; 460 } 461 462 if (VMXNET3_VERSION_GE_7(hw)) { 463 /* start with UPTv2 enabled to avoid ESXi issues */ 464 hw->uptv2_enabled = TRUE; 465 eth_vmxnet3_setup_capabilities(hw, eth_dev); 466 } 467 468 if (hw->used_DCR_capabilities[0] & (1 << VMXNET3_CAP_LARGE_BAR)) { 469 hw->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 470 hw->rx_prod_offset[0] = VMXNET3_REG_LB_RXPROD; 471 hw->rx_prod_offset[1] = VMXNET3_REG_LB_RXPROD2; 472 } else { 473 hw->tx_prod_offset = VMXNET3_REG_TXPROD; 474 hw->rx_prod_offset[0] = VMXNET3_REG_RXPROD; 475 hw->rx_prod_offset[1] = VMXNET3_REG_RXPROD2; 476 } 477 478 /* Getting MAC Address */ 479 mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL); 480 mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH); 481 memcpy(hw->perm_addr, &mac_lo, 4); 482 memcpy(hw->perm_addr + 4, &mac_hi, 2); 483 484 /* Allocate memory for storing MAC addresses */ 485 eth_dev->data->mac_addrs = rte_zmalloc("vmxnet3", RTE_ETHER_ADDR_LEN * 486 VMXNET3_MAX_MAC_ADDRS, 0); 487 if (eth_dev->data->mac_addrs == NULL) { 488 PMD_INIT_LOG(ERR, 489 "Failed to allocate %d bytes needed to store MAC addresses", 490 RTE_ETHER_ADDR_LEN * VMXNET3_MAX_MAC_ADDRS); 491 return -ENOMEM; 492 } 493 /* Copy the permanent MAC address */ 494 rte_ether_addr_copy((struct rte_ether_addr *)hw->perm_addr, 495 ð_dev->data->mac_addrs[0]); 496 497 PMD_INIT_LOG(DEBUG, "MAC Address : " RTE_ETHER_ADDR_PRT_FMT, 498 hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2], 499 hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]); 500 501 /* Put device in Quiesce Mode */ 502 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 503 504 /* allow untagged pkts */ 505 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, 0); 506 507 hw->txdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 508 eth_vmxnet3_txdata_get(hw) : sizeof(struct Vmxnet3_TxDataDesc); 509 510 hw->rxdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 511 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 512 RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) == 513 hw->rxdata_desc_size); 514 515 /* clear shadow stats */ 516 memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats)); 517 memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats)); 518 519 /* clear snapshot stats */ 520 memset(hw->snapshot_tx_stats, 0, sizeof(hw->snapshot_tx_stats)); 521 memset(hw->snapshot_rx_stats, 0, sizeof(hw->snapshot_rx_stats)); 522 523 /* set the initial link status */ 524 memset(&link, 0, sizeof(link)); 525 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 526 link.link_speed = RTE_ETH_SPEED_NUM_10G; 527 link.link_autoneg = RTE_ETH_LINK_FIXED; 528 rte_eth_linkstatus_set(eth_dev, &link); 529 530 return 0; 531 } 532 533 static int 534 eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev) 535 { 536 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 537 538 PMD_INIT_FUNC_TRACE(); 539 540 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 541 return 0; 542 543 if (hw->adapter_stopped == 0) { 544 PMD_INIT_LOG(DEBUG, "Device has not been closed."); 545 return -EBUSY; 546 } 547 548 return 0; 549 } 550 551 static int eth_vmxnet3_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 552 struct rte_pci_device *pci_dev) 553 { 554 return rte_eth_dev_pci_generic_probe(pci_dev, 555 sizeof(struct vmxnet3_hw), eth_vmxnet3_dev_init); 556 } 557 558 static int eth_vmxnet3_pci_remove(struct rte_pci_device *pci_dev) 559 { 560 return rte_eth_dev_pci_generic_remove(pci_dev, eth_vmxnet3_dev_uninit); 561 } 562 563 static struct rte_pci_driver rte_vmxnet3_pmd = { 564 .id_table = pci_id_vmxnet3_map, 565 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 566 .probe = eth_vmxnet3_pci_probe, 567 .remove = eth_vmxnet3_pci_remove, 568 }; 569 570 static void 571 vmxnet3_alloc_intr_resources(struct rte_eth_dev *dev) 572 { 573 struct vmxnet3_hw *hw = dev->data->dev_private; 574 uint32_t cfg; 575 int nvec = 1; /* for link event */ 576 577 /* intr settings */ 578 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 579 VMXNET3_CMD_GET_CONF_INTR); 580 cfg = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 581 hw->intr.type = cfg & 0x3; 582 hw->intr.mask_mode = (cfg >> 2) & 0x3; 583 584 if (hw->intr.type == VMXNET3_IT_AUTO) 585 hw->intr.type = VMXNET3_IT_MSIX; 586 587 if (hw->intr.type == VMXNET3_IT_MSIX) { 588 /* only support shared tx/rx intr */ 589 if (hw->num_tx_queues != hw->num_rx_queues) 590 goto msix_err; 591 592 nvec += hw->num_rx_queues; 593 hw->intr.num_intrs = nvec; 594 return; 595 } 596 597 msix_err: 598 /* the tx/rx queue interrupt will be disabled */ 599 hw->intr.num_intrs = 2; 600 hw->intr.lsc_only = TRUE; 601 PMD_INIT_LOG(INFO, "Enabled MSI-X with %d vectors", hw->intr.num_intrs); 602 } 603 604 static int 605 vmxnet3_dev_configure(struct rte_eth_dev *dev) 606 { 607 const struct rte_memzone *mz; 608 struct vmxnet3_hw *hw = dev->data->dev_private; 609 size_t size; 610 611 PMD_INIT_FUNC_TRACE(); 612 613 if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) 614 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; 615 616 if (!VMXNET3_VERSION_GE_6(hw)) { 617 if (!rte_is_power_of_2(dev->data->nb_rx_queues)) { 618 PMD_INIT_LOG(ERR, 619 "ERROR: Number of rx queues not power of 2"); 620 return -EINVAL; 621 } 622 } 623 624 /* At this point, the number of queues requested has already 625 * been validated against dev_infos max queues by EAL 626 */ 627 if (dev->data->nb_rx_queues > VMXNET3_MAX_RX_QUEUES || 628 dev->data->nb_tx_queues > VMXNET3_MAX_TX_QUEUES) { 629 hw->queuesExtEnabled = 1; 630 } else { 631 hw->queuesExtEnabled = 0; 632 } 633 634 size = dev->data->nb_rx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 635 dev->data->nb_tx_queues * sizeof(struct Vmxnet3_RxQueueDesc); 636 637 if (size > UINT16_MAX) 638 return -EINVAL; 639 640 hw->num_rx_queues = (uint8_t)dev->data->nb_rx_queues; 641 hw->num_tx_queues = (uint8_t)dev->data->nb_tx_queues; 642 643 /* 644 * Allocate a memzone for Vmxnet3_DriverShared - Vmxnet3_DSDevRead 645 * on current socket 646 */ 647 mz = gpa_zone_reserve(dev, sizeof(struct Vmxnet3_DriverShared), 648 "shared", rte_socket_id(), 8, 1); 649 650 if (mz == NULL) { 651 PMD_INIT_LOG(ERR, "ERROR: Creating shared zone"); 652 return -ENOMEM; 653 } 654 memset(mz->addr, 0, mz->len); 655 656 hw->shared = mz->addr; 657 hw->sharedPA = mz->iova; 658 659 /* 660 * Allocate a memzone for Vmxnet3_RxQueueDesc - Vmxnet3_TxQueueDesc 661 * on current socket. 662 * 663 * We cannot reuse this memzone from previous allocation as its size 664 * depends on the number of tx and rx queues, which could be different 665 * from one config to another. 666 */ 667 mz = gpa_zone_reserve(dev, size, "queuedesc", rte_socket_id(), 668 VMXNET3_QUEUE_DESC_ALIGN, 0); 669 if (mz == NULL) { 670 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); 671 return -ENOMEM; 672 } 673 memset(mz->addr, 0, mz->len); 674 675 hw->tqd_start = (Vmxnet3_TxQueueDesc *)mz->addr; 676 hw->rqd_start = (Vmxnet3_RxQueueDesc *)(hw->tqd_start + hw->num_tx_queues); 677 678 hw->queueDescPA = mz->iova; 679 hw->queue_desc_len = (uint16_t)size; 680 681 if (dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 682 /* Allocate memory structure for UPT1_RSSConf and configure */ 683 mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf), 684 "rss_conf", rte_socket_id(), 685 RTE_CACHE_LINE_SIZE, 1); 686 if (mz == NULL) { 687 PMD_INIT_LOG(ERR, 688 "ERROR: Creating rss_conf structure zone"); 689 return -ENOMEM; 690 } 691 memset(mz->addr, 0, mz->len); 692 693 hw->rss_conf = mz->addr; 694 hw->rss_confPA = mz->iova; 695 } 696 697 vmxnet3_alloc_intr_resources(dev); 698 699 return 0; 700 } 701 702 static void 703 vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr) 704 { 705 uint32_t val; 706 707 PMD_INIT_LOG(DEBUG, 708 "Writing MAC Address : " RTE_ETHER_ADDR_PRT_FMT, 709 addr[0], addr[1], addr[2], 710 addr[3], addr[4], addr[5]); 711 712 memcpy(&val, addr, 4); 713 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACL, val); 714 715 memcpy(&val, addr + 4, 2); 716 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val); 717 } 718 719 /* 720 * Configure the hardware to generate MSI-X interrupts. 721 * If setting up MSIx fails, try setting up MSI (only 1 interrupt vector 722 * which will be disabled to allow lsc to work). 723 * 724 * Returns 0 on success and -1 otherwise. 725 */ 726 static int 727 vmxnet3_configure_msix(struct rte_eth_dev *dev) 728 { 729 struct vmxnet3_hw *hw = dev->data->dev_private; 730 struct rte_intr_handle *intr_handle = dev->intr_handle; 731 uint16_t intr_vector; 732 int i; 733 734 hw->intr.event_intr_idx = 0; 735 736 /* only vfio-pci driver can support interrupt mode. */ 737 if (!rte_intr_cap_multiple(intr_handle) || 738 dev->data->dev_conf.intr_conf.rxq == 0) 739 return -1; 740 741 intr_vector = dev->data->nb_rx_queues; 742 if (intr_vector > MAX_RX_QUEUES(hw)) { 743 PMD_INIT_LOG(ERR, "At most %d intr queues supported", 744 MAX_RX_QUEUES(hw)); 745 return -ENOTSUP; 746 } 747 748 if (rte_intr_efd_enable(intr_handle, intr_vector)) { 749 PMD_INIT_LOG(ERR, "Failed to enable fastpath event fd"); 750 return -1; 751 } 752 753 if (rte_intr_dp_is_en(intr_handle)) { 754 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec", 755 dev->data->nb_rx_queues)) { 756 PMD_INIT_LOG(ERR, "Failed to allocate %d Rx queues intr_vec", 757 dev->data->nb_rx_queues); 758 rte_intr_efd_disable(intr_handle); 759 return -ENOMEM; 760 } 761 } 762 763 if (!rte_intr_allow_others(intr_handle) && 764 dev->data->dev_conf.intr_conf.lsc != 0) { 765 PMD_INIT_LOG(ERR, "not enough intr vector to support both Rx interrupt and LSC"); 766 rte_intr_vec_list_free(intr_handle); 767 rte_intr_efd_disable(intr_handle); 768 return -1; 769 } 770 771 /* if we cannot allocate one MSI-X vector per queue, don't enable 772 * interrupt mode. 773 */ 774 if (hw->intr.num_intrs != 775 (rte_intr_nb_efd_get(intr_handle) + 1)) { 776 PMD_INIT_LOG(ERR, "Device configured with %d Rx intr vectors, expecting %d", 777 hw->intr.num_intrs, 778 rte_intr_nb_efd_get(intr_handle) + 1); 779 rte_intr_vec_list_free(intr_handle); 780 rte_intr_efd_disable(intr_handle); 781 return -1; 782 } 783 784 for (i = 0; i < dev->data->nb_rx_queues; i++) 785 if (rte_intr_vec_list_index_set(intr_handle, i, i + 1)) 786 return -rte_errno; 787 788 for (i = 0; i < hw->intr.num_intrs; i++) 789 hw->intr.mod_levels[i] = UPT1_IML_ADAPTIVE; 790 791 PMD_INIT_LOG(INFO, "intr type %u, mode %u, %u vectors allocated", 792 hw->intr.type, hw->intr.mask_mode, hw->intr.num_intrs); 793 794 return 0; 795 } 796 797 static int 798 vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev) 799 { 800 struct vmxnet3_hw *hw = dev->data->dev_private; 801 Vmxnet3_DriverShared *shared = hw->shared; 802 Vmxnet3_CmdInfo *cmdInfo; 803 struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES]; 804 uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES]; 805 uint32_t num, i, j, size; 806 807 if (hw->memRegsPA == 0) { 808 const struct rte_memzone *mz; 809 810 size = sizeof(Vmxnet3_MemRegs) + 811 (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) * 812 sizeof(Vmxnet3_MemoryRegion); 813 814 mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8, 815 1); 816 if (mz == NULL) { 817 PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone"); 818 return -ENOMEM; 819 } 820 memset(mz->addr, 0, mz->len); 821 hw->memRegs = mz->addr; 822 hw->memRegsPA = mz->iova; 823 } 824 825 num = hw->num_rx_queues; 826 827 for (i = 0; i < num; i++) { 828 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 829 830 mp[i] = rxq->mp; 831 index[i] = 1 << i; 832 } 833 834 /* 835 * The same mempool could be used by multiple queues. In such a case, 836 * remove duplicate mempool entries. Only one entry is kept with 837 * bitmask indicating queues that are using this mempool. 838 */ 839 for (i = 1; i < num; i++) { 840 for (j = 0; j < i; j++) { 841 if (mp[i] == mp[j]) { 842 mp[i] = NULL; 843 index[j] |= 1 << i; 844 break; 845 } 846 } 847 } 848 849 j = 0; 850 for (i = 0; i < num; i++) { 851 if (mp[i] == NULL) 852 continue; 853 854 Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j]; 855 856 mr->startPA = 857 (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->iova; 858 mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ? 859 STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX; 860 mr->txQueueBits = index[i]; 861 mr->rxQueueBits = index[i]; 862 863 PMD_INIT_LOG(INFO, 864 "index: %u startPA: %" PRIu64 " length: %u, " 865 "rxBits: %x", 866 j, mr->startPA, mr->length, mr->rxQueueBits); 867 j++; 868 } 869 hw->memRegs->numRegs = j; 870 PMD_INIT_LOG(INFO, "numRegs: %u", j); 871 872 size = sizeof(Vmxnet3_MemRegs) + 873 (j - 1) * sizeof(Vmxnet3_MemoryRegion); 874 875 cmdInfo = &shared->cu.cmdInfo; 876 cmdInfo->varConf.confVer = 1; 877 cmdInfo->varConf.confLen = size; 878 cmdInfo->varConf.confPA = hw->memRegsPA; 879 880 return 0; 881 } 882 883 static int 884 vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) 885 { 886 struct rte_eth_conf port_conf = dev->data->dev_conf; 887 struct vmxnet3_hw *hw = dev->data->dev_private; 888 struct rte_intr_handle *intr_handle = dev->intr_handle; 889 uint32_t mtu = dev->data->mtu; 890 Vmxnet3_DriverShared *shared = hw->shared; 891 Vmxnet3_DSDevRead *devRead = &shared->devRead; 892 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 893 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 894 uint32_t i; 895 int ret; 896 897 hw->mtu = mtu; 898 899 shared->magic = VMXNET3_REV1_MAGIC; 900 devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM; 901 902 /* Setting up Guest OS information */ 903 devRead->misc.driverInfo.gos.gosBits = sizeof(void *) == 4 ? 904 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64; 905 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 906 devRead->misc.driverInfo.vmxnet3RevSpt = 1; 907 devRead->misc.driverInfo.uptVerSpt = 1; 908 909 devRead->misc.mtu = rte_le_to_cpu_32(mtu); 910 devRead->misc.queueDescPA = hw->queueDescPA; 911 devRead->misc.queueDescLen = hw->queue_desc_len; 912 devRead->misc.numTxQueues = hw->num_tx_queues; 913 devRead->misc.numRxQueues = hw->num_rx_queues; 914 915 for (i = 0; i < hw->num_tx_queues; i++) { 916 Vmxnet3_TxQueueDesc *tqd = &hw->tqd_start[i]; 917 vmxnet3_tx_queue_t *txq = dev->data->tx_queues[i]; 918 919 txq->shared = &hw->tqd_start[i]; 920 921 tqd->ctrl.txNumDeferred = 0; 922 tqd->ctrl.txThreshold = 1; 923 tqd->conf.txRingBasePA = txq->cmd_ring.basePA; 924 tqd->conf.compRingBasePA = txq->comp_ring.basePA; 925 tqd->conf.dataRingBasePA = txq->data_ring.basePA; 926 927 tqd->conf.txRingSize = txq->cmd_ring.size; 928 tqd->conf.compRingSize = txq->comp_ring.size; 929 tqd->conf.dataRingSize = txq->data_ring.size; 930 tqd->conf.txDataRingDescSize = txq->txdata_desc_size; 931 932 if (hw->intr.lsc_only) 933 tqd->conf.intrIdx = 1; 934 else 935 tqd->conf.intrIdx = 936 rte_intr_vec_list_index_get(intr_handle, 937 i); 938 tqd->status.stopped = TRUE; 939 tqd->status.error = 0; 940 memset(&tqd->stats, 0, sizeof(tqd->stats)); 941 } 942 943 for (i = 0; i < hw->num_rx_queues; i++) { 944 Vmxnet3_RxQueueDesc *rqd = &hw->rqd_start[i]; 945 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 946 947 rxq->shared = &hw->rqd_start[i]; 948 949 rqd->conf.rxRingBasePA[0] = rxq->cmd_ring[0].basePA; 950 rqd->conf.rxRingBasePA[1] = rxq->cmd_ring[1].basePA; 951 rqd->conf.compRingBasePA = rxq->comp_ring.basePA; 952 953 rqd->conf.rxRingSize[0] = rxq->cmd_ring[0].size; 954 rqd->conf.rxRingSize[1] = rxq->cmd_ring[1].size; 955 rqd->conf.compRingSize = rxq->comp_ring.size; 956 957 if (VMXNET3_VERSION_GE_3(hw)) { 958 rqd->conf.rxDataRingBasePA = rxq->data_ring.basePA; 959 rqd->conf.rxDataRingDescSize = rxq->data_desc_size; 960 } 961 962 if (hw->intr.lsc_only) 963 rqd->conf.intrIdx = 1; 964 else 965 rqd->conf.intrIdx = 966 rte_intr_vec_list_index_get(intr_handle, 967 i); 968 rqd->status.stopped = TRUE; 969 rqd->status.error = 0; 970 memset(&rqd->stats, 0, sizeof(rqd->stats)); 971 } 972 973 /* intr settings */ 974 if (VMXNET3_VERSION_GE_6(hw) && hw->queuesExtEnabled) { 975 devReadExt->intrConfExt.autoMask = hw->intr.mask_mode == 976 VMXNET3_IMM_AUTO; 977 devReadExt->intrConfExt.numIntrs = hw->intr.num_intrs; 978 for (i = 0; i < hw->intr.num_intrs; i++) 979 devReadExt->intrConfExt.modLevels[i] = 980 hw->intr.mod_levels[i]; 981 982 devReadExt->intrConfExt.eventIntrIdx = hw->intr.event_intr_idx; 983 devReadExt->intrConfExt.intrCtrl |= 984 rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 985 } else { 986 devRead->intrConf.autoMask = hw->intr.mask_mode == 987 VMXNET3_IMM_AUTO; 988 devRead->intrConf.numIntrs = hw->intr.num_intrs; 989 for (i = 0; i < hw->intr.num_intrs; i++) 990 devRead->intrConf.modLevels[i] = hw->intr.mod_levels[i]; 991 992 devRead->intrConf.eventIntrIdx = hw->intr.event_intr_idx; 993 devRead->intrConf.intrCtrl |= rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 994 } 995 996 /* RxMode set to 0 of VMXNET3_RXM_xxx */ 997 devRead->rxFilterConf.rxMode = 0; 998 999 /* Setting up feature flags */ 1000 if (rx_offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 1001 devRead->misc.uptFeatures |= VMXNET3_F_RXCSUM; 1002 1003 if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) { 1004 devRead->misc.uptFeatures |= VMXNET3_F_LRO; 1005 devRead->misc.maxNumRxSG = 0; 1006 } 1007 1008 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 1009 ret = vmxnet3_rss_configure(dev); 1010 if (ret != VMXNET3_SUCCESS) 1011 return ret; 1012 1013 devRead->misc.uptFeatures |= VMXNET3_F_RSS; 1014 devRead->rssConfDesc.confVer = 1; 1015 devRead->rssConfDesc.confLen = sizeof(struct VMXNET3_RSSConf); 1016 devRead->rssConfDesc.confPA = hw->rss_confPA; 1017 } 1018 1019 ret = vmxnet3_dev_vlan_offload_set(dev, 1020 RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK); 1021 if (ret) 1022 return ret; 1023 1024 vmxnet3_write_mac(hw, dev->data->mac_addrs->addr_bytes); 1025 1026 return VMXNET3_SUCCESS; 1027 } 1028 1029 static void 1030 vmxnet3_init_bufsize(struct vmxnet3_hw *hw) 1031 { 1032 struct Vmxnet3_DriverShared *shared = hw->shared; 1033 union Vmxnet3_CmdInfo *cmd_info = &shared->cu.cmdInfo; 1034 1035 if (!VMXNET3_VERSION_GE_7(hw)) 1036 return; 1037 1038 cmd_info->ringBufSize.ring1BufSizeType0 = hw->rxdata_buf_size; 1039 cmd_info->ringBufSize.ring1BufSizeType1 = 0; 1040 cmd_info->ringBufSize.ring2BufSizeType1 = hw->rxdata_buf_size; 1041 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1042 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 1043 } 1044 1045 /* 1046 * Configure device link speed and setup link. 1047 * Must be called after eth_vmxnet3_dev_init. Other wise it might fail 1048 * It returns 0 on success. 1049 */ 1050 static int 1051 vmxnet3_dev_start(struct rte_eth_dev *dev) 1052 { 1053 int ret; 1054 struct vmxnet3_hw *hw = dev->data->dev_private; 1055 uint16_t i; 1056 1057 PMD_INIT_FUNC_TRACE(); 1058 1059 /* Save stats before it is reset by CMD_ACTIVATE */ 1060 vmxnet3_hw_stats_save(hw); 1061 1062 /* configure MSI-X */ 1063 ret = vmxnet3_configure_msix(dev); 1064 if (ret < 0) { 1065 /* revert to lsc only */ 1066 hw->intr.num_intrs = 2; 1067 hw->intr.lsc_only = TRUE; 1068 } 1069 1070 ret = vmxnet3_setup_driver_shared(dev); 1071 if (ret != VMXNET3_SUCCESS) 1072 return ret; 1073 1074 /* Exchange shared data with device */ 1075 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 1076 VMXNET3_GET_ADDR_LO(hw->sharedPA)); 1077 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 1078 VMXNET3_GET_ADDR_HI(hw->sharedPA)); 1079 1080 /* Activate device by register write */ 1081 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV); 1082 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1083 1084 if (ret != 0) { 1085 PMD_INIT_LOG(ERR, "Device activation: UNSUCCESSFUL"); 1086 return -EINVAL; 1087 } 1088 1089 /* Check memregs restrictions first */ 1090 if (dev->data->nb_rx_queues <= VMXNET3_MAX_RX_QUEUES && 1091 dev->data->nb_tx_queues <= VMXNET3_MAX_TX_QUEUES) { 1092 ret = vmxnet3_dev_setup_memreg(dev); 1093 if (ret == 0) { 1094 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1095 VMXNET3_CMD_REGISTER_MEMREGS); 1096 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1097 if (ret != 0) 1098 PMD_INIT_LOG(DEBUG, 1099 "Failed in setup memory region cmd\n"); 1100 ret = 0; 1101 } else { 1102 PMD_INIT_LOG(DEBUG, "Failed to setup memory region\n"); 1103 } 1104 } else { 1105 PMD_INIT_LOG(WARNING, "Memregs can't init (rx: %d, tx: %d)", 1106 dev->data->nb_rx_queues, dev->data->nb_tx_queues); 1107 } 1108 1109 if (VMXNET3_VERSION_GE_4(hw) && 1110 dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 1111 /* Check for additional RSS */ 1112 ret = vmxnet3_v4_rss_configure(dev); 1113 if (ret != VMXNET3_SUCCESS) { 1114 PMD_INIT_LOG(ERR, "Failed to configure v4 RSS"); 1115 return ret; 1116 } 1117 } 1118 1119 /* 1120 * Load RX queues with blank mbufs and update next2fill index for device 1121 * Update RxMode of the device 1122 */ 1123 ret = vmxnet3_dev_rxtx_init(dev); 1124 if (ret != VMXNET3_SUCCESS) { 1125 PMD_INIT_LOG(ERR, "Device queue init: UNSUCCESSFUL"); 1126 return ret; 1127 } 1128 1129 vmxnet3_init_bufsize(hw); 1130 1131 hw->adapter_stopped = FALSE; 1132 1133 /* Setting proper Rx Mode and issue Rx Mode Update command */ 1134 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); 1135 1136 #ifndef RTE_EXEC_ENV_FREEBSD 1137 /* Setup interrupt callback */ 1138 rte_intr_callback_register(dev->intr_handle, 1139 vmxnet3_interrupt_handler, dev); 1140 1141 if (rte_intr_enable(dev->intr_handle) < 0) { 1142 PMD_INIT_LOG(ERR, "interrupt enable failed"); 1143 return -EIO; 1144 } 1145 1146 /* enable all intrs */ 1147 vmxnet3_enable_all_intrs(hw); 1148 #endif 1149 1150 vmxnet3_process_events(dev); 1151 1152 /* 1153 * Update link state from device since this won't be 1154 * done upon starting with lsc in use. This is done 1155 * only after enabling interrupts to avoid any race 1156 * where the link state could change without an 1157 * interrupt being fired. 1158 */ 1159 __vmxnet3_dev_link_update(dev, 0); 1160 1161 for (i = 0; i < dev->data->nb_rx_queues; i++) 1162 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 1163 for (i = 0; i < dev->data->nb_tx_queues; i++) 1164 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 1165 1166 return VMXNET3_SUCCESS; 1167 } 1168 1169 /* 1170 * Stop device: disable rx and tx functions to allow for reconfiguring. 1171 */ 1172 static int 1173 vmxnet3_dev_stop(struct rte_eth_dev *dev) 1174 { 1175 struct rte_eth_link link; 1176 struct vmxnet3_hw *hw = dev->data->dev_private; 1177 struct rte_intr_handle *intr_handle = dev->intr_handle; 1178 uint16_t i; 1179 int ret; 1180 1181 PMD_INIT_FUNC_TRACE(); 1182 1183 if (hw->adapter_stopped == 1) { 1184 PMD_INIT_LOG(DEBUG, "Device already stopped."); 1185 return 0; 1186 } 1187 1188 do { 1189 /* Unregister has lock to make sure there is no running cb. 1190 * This has to happen first since vmxnet3_interrupt_handler 1191 * reenables interrupts by calling vmxnet3_enable_intr 1192 */ 1193 ret = rte_intr_callback_unregister(intr_handle, 1194 vmxnet3_interrupt_handler, 1195 (void *)-1); 1196 } while (ret == -EAGAIN); 1197 1198 if (ret < 0) 1199 PMD_DRV_LOG(ERR, "Error attempting to unregister intr cb: %d", 1200 ret); 1201 1202 PMD_INIT_LOG(DEBUG, "Disabled %d intr callbacks", ret); 1203 1204 /* disable interrupts */ 1205 vmxnet3_disable_all_intrs(hw); 1206 1207 rte_intr_disable(intr_handle); 1208 1209 /* Clean datapath event and queue/vector mapping */ 1210 rte_intr_efd_disable(intr_handle); 1211 rte_intr_vec_list_free(intr_handle); 1212 1213 /* quiesce the device first */ 1214 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 1215 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0); 1216 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 0); 1217 1218 /* reset the device */ 1219 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 1220 PMD_INIT_LOG(DEBUG, "Device reset."); 1221 1222 vmxnet3_dev_clear_queues(dev); 1223 1224 /* Clear recorded link status */ 1225 memset(&link, 0, sizeof(link)); 1226 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 1227 link.link_speed = RTE_ETH_SPEED_NUM_10G; 1228 link.link_autoneg = RTE_ETH_LINK_FIXED; 1229 rte_eth_linkstatus_set(dev, &link); 1230 1231 hw->adapter_stopped = 1; 1232 dev->data->dev_started = 0; 1233 1234 for (i = 0; i < dev->data->nb_rx_queues; i++) 1235 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1236 for (i = 0; i < dev->data->nb_tx_queues; i++) 1237 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1238 1239 return 0; 1240 } 1241 1242 static void 1243 vmxnet3_free_queues(struct rte_eth_dev *dev) 1244 { 1245 int i; 1246 1247 PMD_INIT_FUNC_TRACE(); 1248 1249 for (i = 0; i < dev->data->nb_rx_queues; i++) 1250 vmxnet3_dev_rx_queue_release(dev, i); 1251 dev->data->nb_rx_queues = 0; 1252 1253 for (i = 0; i < dev->data->nb_tx_queues; i++) 1254 vmxnet3_dev_tx_queue_release(dev, i); 1255 dev->data->nb_tx_queues = 0; 1256 } 1257 1258 /* 1259 * Reset and stop device. 1260 */ 1261 static int 1262 vmxnet3_dev_close(struct rte_eth_dev *dev) 1263 { 1264 int ret; 1265 PMD_INIT_FUNC_TRACE(); 1266 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1267 return 0; 1268 1269 ret = vmxnet3_dev_stop(dev); 1270 vmxnet3_free_queues(dev); 1271 1272 return ret; 1273 } 1274 1275 static int 1276 vmxnet3_dev_reset(struct rte_eth_dev *dev) 1277 { 1278 int ret; 1279 1280 ret = eth_vmxnet3_dev_uninit(dev); 1281 if (ret) 1282 return ret; 1283 ret = eth_vmxnet3_dev_init(dev); 1284 return ret; 1285 } 1286 1287 static void 1288 vmxnet3_hw_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1289 struct UPT1_TxStats *res) 1290 { 1291 #define VMXNET3_UPDATE_TX_STAT(h, i, f, r) \ 1292 ((r)->f = (h)->tqd_start[(i)].stats.f + \ 1293 (h)->saved_tx_stats[(i)].f) 1294 1295 VMXNET3_UPDATE_TX_STAT(hw, q, ucastPktsTxOK, res); 1296 VMXNET3_UPDATE_TX_STAT(hw, q, mcastPktsTxOK, res); 1297 VMXNET3_UPDATE_TX_STAT(hw, q, bcastPktsTxOK, res); 1298 VMXNET3_UPDATE_TX_STAT(hw, q, ucastBytesTxOK, res); 1299 VMXNET3_UPDATE_TX_STAT(hw, q, mcastBytesTxOK, res); 1300 VMXNET3_UPDATE_TX_STAT(hw, q, bcastBytesTxOK, res); 1301 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxError, res); 1302 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxDiscard, res); 1303 1304 #undef VMXNET3_UPDATE_TX_STAT 1305 } 1306 1307 static void 1308 vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1309 struct UPT1_RxStats *res) 1310 { 1311 #define VMXNET3_UPDATE_RX_STAT(h, i, f, r) \ 1312 ((r)->f = (h)->rqd_start[(i)].stats.f + \ 1313 (h)->saved_rx_stats[(i)].f) 1314 1315 VMXNET3_UPDATE_RX_STAT(hw, q, ucastPktsRxOK, res); 1316 VMXNET3_UPDATE_RX_STAT(hw, q, mcastPktsRxOK, res); 1317 VMXNET3_UPDATE_RX_STAT(hw, q, bcastPktsRxOK, res); 1318 VMXNET3_UPDATE_RX_STAT(hw, q, ucastBytesRxOK, res); 1319 VMXNET3_UPDATE_RX_STAT(hw, q, mcastBytesRxOK, res); 1320 VMXNET3_UPDATE_RX_STAT(hw, q, bcastBytesRxOK, res); 1321 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res); 1322 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res); 1323 1324 #undef VMXNET3_UPDATE_RX_STAT 1325 } 1326 1327 static void 1328 vmxnet3_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1329 struct UPT1_TxStats *res) 1330 { 1331 vmxnet3_hw_tx_stats_get(hw, q, res); 1332 1333 #define VMXNET3_REDUCE_SNAPSHOT_TX_STAT(h, i, f, r) \ 1334 ((r)->f -= (h)->snapshot_tx_stats[(i)].f) 1335 1336 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastPktsTxOK, res); 1337 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastPktsTxOK, res); 1338 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastPktsTxOK, res); 1339 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastBytesTxOK, res); 1340 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastBytesTxOK, res); 1341 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastBytesTxOK, res); 1342 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxError, res); 1343 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxDiscard, res); 1344 1345 #undef VMXNET3_REDUCE_SNAPSHOT_TX_STAT 1346 } 1347 1348 static void 1349 vmxnet3_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1350 struct UPT1_RxStats *res) 1351 { 1352 vmxnet3_hw_rx_stats_get(hw, q, res); 1353 1354 #define VMXNET3_REDUCE_SNAPSHOT_RX_STAT(h, i, f, r) \ 1355 ((r)->f -= (h)->snapshot_rx_stats[(i)].f) 1356 1357 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastPktsRxOK, res); 1358 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastPktsRxOK, res); 1359 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastPktsRxOK, res); 1360 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastBytesRxOK, res); 1361 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastBytesRxOK, res); 1362 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastBytesRxOK, res); 1363 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxError, res); 1364 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxOutOfBuf, res); 1365 1366 #undef VMXNET3_REDUCE_SNAPSHOT_RX_STAT 1367 } 1368 1369 static void 1370 vmxnet3_hw_stats_save(struct vmxnet3_hw *hw) 1371 { 1372 unsigned int i; 1373 1374 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1375 1376 for (i = 0; i < hw->num_tx_queues; i++) 1377 vmxnet3_hw_tx_stats_get(hw, i, &hw->saved_tx_stats[i]); 1378 for (i = 0; i < hw->num_rx_queues; i++) 1379 vmxnet3_hw_rx_stats_get(hw, i, &hw->saved_rx_stats[i]); 1380 } 1381 1382 static int 1383 vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 1384 struct rte_eth_xstat_name *xstats_names, 1385 unsigned int n) 1386 { 1387 unsigned int i, t, count = 0; 1388 unsigned int nstats = 1389 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 1390 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 1391 1392 if (!xstats_names || n < nstats) 1393 return nstats; 1394 1395 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1396 if (!dev->data->rx_queues[i]) 1397 continue; 1398 1399 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 1400 snprintf(xstats_names[count].name, 1401 sizeof(xstats_names[count].name), 1402 "rx_q%u_%s", i, 1403 vmxnet3_rxq_stat_strings[t].name); 1404 count++; 1405 } 1406 } 1407 1408 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1409 if (!dev->data->tx_queues[i]) 1410 continue; 1411 1412 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 1413 snprintf(xstats_names[count].name, 1414 sizeof(xstats_names[count].name), 1415 "tx_q%u_%s", i, 1416 vmxnet3_txq_stat_strings[t].name); 1417 count++; 1418 } 1419 } 1420 1421 return count; 1422 } 1423 1424 static int 1425 vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1426 unsigned int n) 1427 { 1428 unsigned int i, t, count = 0; 1429 unsigned int nstats = 1430 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 1431 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 1432 1433 if (n < nstats) 1434 return nstats; 1435 1436 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1437 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i]; 1438 1439 if (rxq == NULL) 1440 continue; 1441 1442 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 1443 xstats[count].value = *(uint64_t *)(((char *)&rxq->stats) + 1444 vmxnet3_rxq_stat_strings[t].offset); 1445 xstats[count].id = count; 1446 count++; 1447 } 1448 } 1449 1450 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1451 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; 1452 1453 if (txq == NULL) 1454 continue; 1455 1456 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 1457 xstats[count].value = *(uint64_t *)(((char *)&txq->stats) + 1458 vmxnet3_txq_stat_strings[t].offset); 1459 xstats[count].id = count; 1460 count++; 1461 } 1462 } 1463 1464 return count; 1465 } 1466 1467 static int 1468 vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1469 { 1470 unsigned int i; 1471 struct vmxnet3_hw *hw = dev->data->dev_private; 1472 struct UPT1_TxStats txStats; 1473 struct UPT1_RxStats rxStats; 1474 1475 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1476 1477 for (i = 0; i < hw->num_tx_queues; i++) { 1478 vmxnet3_tx_stats_get(hw, i, &txStats); 1479 1480 stats->q_opackets[i] = txStats.ucastPktsTxOK + 1481 txStats.mcastPktsTxOK + 1482 txStats.bcastPktsTxOK; 1483 1484 stats->q_obytes[i] = txStats.ucastBytesTxOK + 1485 txStats.mcastBytesTxOK + 1486 txStats.bcastBytesTxOK; 1487 1488 stats->opackets += stats->q_opackets[i]; 1489 stats->obytes += stats->q_obytes[i]; 1490 stats->oerrors += txStats.pktsTxError + txStats.pktsTxDiscard; 1491 } 1492 1493 for (i = 0; i < hw->num_rx_queues; i++) { 1494 vmxnet3_rx_stats_get(hw, i, &rxStats); 1495 1496 stats->q_ipackets[i] = rxStats.ucastPktsRxOK + 1497 rxStats.mcastPktsRxOK + 1498 rxStats.bcastPktsRxOK; 1499 1500 stats->q_ibytes[i] = rxStats.ucastBytesRxOK + 1501 rxStats.mcastBytesRxOK + 1502 rxStats.bcastBytesRxOK; 1503 1504 stats->ipackets += stats->q_ipackets[i]; 1505 stats->ibytes += stats->q_ibytes[i]; 1506 1507 stats->q_errors[i] = rxStats.pktsRxError; 1508 stats->ierrors += rxStats.pktsRxError; 1509 stats->imissed += rxStats.pktsRxOutOfBuf; 1510 } 1511 1512 return 0; 1513 } 1514 1515 static int 1516 vmxnet3_dev_stats_reset(struct rte_eth_dev *dev) 1517 { 1518 unsigned int i; 1519 struct vmxnet3_hw *hw = dev->data->dev_private; 1520 struct UPT1_TxStats txStats = {0}; 1521 struct UPT1_RxStats rxStats = {0}; 1522 1523 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1524 1525 RTE_BUILD_BUG_ON(RTE_ETHDEV_QUEUE_STAT_CNTRS < VMXNET3_MAX_TX_QUEUES); 1526 1527 for (i = 0; i < hw->num_tx_queues; i++) { 1528 vmxnet3_hw_tx_stats_get(hw, i, &txStats); 1529 memcpy(&hw->snapshot_tx_stats[i], &txStats, 1530 sizeof(hw->snapshot_tx_stats[0])); 1531 } 1532 for (i = 0; i < hw->num_rx_queues; i++) { 1533 vmxnet3_hw_rx_stats_get(hw, i, &rxStats); 1534 memcpy(&hw->snapshot_rx_stats[i], &rxStats, 1535 sizeof(hw->snapshot_rx_stats[0])); 1536 } 1537 1538 return 0; 1539 } 1540 1541 static int 1542 vmxnet3_dev_info_get(struct rte_eth_dev *dev, 1543 struct rte_eth_dev_info *dev_info) 1544 { 1545 struct vmxnet3_hw *hw = dev->data->dev_private; 1546 int queues = 0; 1547 1548 if (VMXNET3_VERSION_GE_6(hw)) { 1549 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1550 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 1551 queues = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1552 1553 if (queues > 0) { 1554 dev_info->max_rx_queues = 1555 RTE_MIN(VMXNET3_EXT_MAX_RX_QUEUES, ((queues >> 8) & 0xff)); 1556 dev_info->max_tx_queues = 1557 RTE_MIN(VMXNET3_EXT_MAX_TX_QUEUES, (queues & 0xff)); 1558 } else { 1559 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; 1560 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; 1561 } 1562 } else { 1563 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; 1564 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; 1565 } 1566 1567 dev_info->min_rx_bufsize = 1518 + RTE_PKTMBUF_HEADROOM; 1568 dev_info->max_rx_pktlen = 16384; /* includes CRC, cf MAXFRS register */ 1569 dev_info->min_mtu = VMXNET3_MIN_MTU; 1570 dev_info->max_mtu = VMXNET3_MAX_MTU; 1571 dev_info->speed_capa = RTE_ETH_LINK_SPEED_10G; 1572 dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS; 1573 1574 dev_info->flow_type_rss_offloads = VMXNET3_RSS_OFFLOAD_ALL; 1575 1576 if (VMXNET3_VERSION_GE_4(hw)) { 1577 dev_info->flow_type_rss_offloads |= VMXNET3_V4_RSS_MASK; 1578 } 1579 1580 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 1581 .nb_max = VMXNET3_RX_RING_MAX_SIZE, 1582 .nb_min = VMXNET3_DEF_RX_RING_SIZE, 1583 .nb_align = 1, 1584 }; 1585 1586 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 1587 .nb_max = VMXNET3_TX_RING_MAX_SIZE, 1588 .nb_min = VMXNET3_DEF_TX_RING_SIZE, 1589 .nb_align = 1, 1590 .nb_seg_max = VMXNET3_TX_MAX_SEG, 1591 .nb_mtu_seg_max = VMXNET3_MAX_TXD_PER_PKT, 1592 }; 1593 1594 dev_info->rx_offload_capa = VMXNET3_RX_OFFLOAD_CAP; 1595 dev_info->rx_queue_offload_capa = 0; 1596 dev_info->tx_offload_capa = VMXNET3_TX_OFFLOAD_CAP; 1597 dev_info->tx_queue_offload_capa = 0; 1598 if (hw->rss_conf == NULL) { 1599 /* RSS not configured */ 1600 dev_info->reta_size = 0; 1601 } else { 1602 dev_info->reta_size = hw->rss_conf->indTableSize; 1603 } 1604 return 0; 1605 } 1606 1607 static int 1608 vmxnet3_hw_ver_get(struct rte_eth_dev *dev, 1609 char *fw_version, size_t fw_size) 1610 { 1611 int ret; 1612 struct vmxnet3_hw *hw = dev->data->dev_private; 1613 1614 ret = snprintf(fw_version, fw_size, "v%d", hw->version); 1615 1616 ret += 1; /* add the size of '\0' */ 1617 if (fw_size < (uint32_t)ret) 1618 return ret; 1619 else 1620 return 0; 1621 } 1622 1623 static const uint32_t * 1624 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev, 1625 size_t *no_of_elements) 1626 { 1627 static const uint32_t ptypes[] = { 1628 RTE_PTYPE_L3_IPV4_EXT, 1629 RTE_PTYPE_L3_IPV4, 1630 }; 1631 1632 if (dev->rx_pkt_burst == vmxnet3_recv_pkts) { 1633 *no_of_elements = RTE_DIM(ptypes); 1634 return ptypes; 1635 } 1636 return NULL; 1637 } 1638 1639 static int 1640 vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) 1641 { 1642 struct vmxnet3_hw *hw = dev->data->dev_private; 1643 1644 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)(hw->perm_addr)); 1645 vmxnet3_write_mac(hw, mac_addr->addr_bytes); 1646 return 0; 1647 } 1648 1649 static int 1650 vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1651 { 1652 struct vmxnet3_hw *hw = dev->data->dev_private; 1653 uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 4; 1654 1655 if (mtu < VMXNET3_MIN_MTU) 1656 return -EINVAL; 1657 1658 if (VMXNET3_VERSION_GE_6(hw)) { 1659 if (mtu > VMXNET3_V6_MAX_MTU) 1660 return -EINVAL; 1661 } else { 1662 if (mtu > VMXNET3_MAX_MTU) { 1663 PMD_DRV_LOG(ERR, "MTU %d too large in device version v%d", 1664 mtu, hw->version); 1665 return -EINVAL; 1666 } 1667 } 1668 1669 dev->data->mtu = mtu; 1670 /* update max frame size */ 1671 dev->data->dev_conf.rxmode.mtu = frame_size; 1672 1673 if (dev->data->dev_started == 0) 1674 return 0; 1675 1676 /* changing mtu for vmxnet3 pmd does not require a restart 1677 * as it does not need to repopulate the rx rings to support 1678 * different mtu size. We stop and restart the device here 1679 * just to pass the mtu info to the backend. 1680 */ 1681 vmxnet3_dev_stop(dev); 1682 vmxnet3_dev_start(dev); 1683 1684 return 0; 1685 } 1686 1687 /* return 0 means link status changed, -1 means not changed */ 1688 static int 1689 __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 1690 __rte_unused int wait_to_complete) 1691 { 1692 struct vmxnet3_hw *hw = dev->data->dev_private; 1693 struct rte_eth_link link; 1694 uint32_t ret; 1695 1696 memset(&link, 0, sizeof(link)); 1697 1698 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 1699 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1700 1701 if (ret & 0x1) 1702 link.link_status = RTE_ETH_LINK_UP; 1703 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 1704 link.link_speed = RTE_ETH_SPEED_NUM_10G; 1705 link.link_autoneg = RTE_ETH_LINK_FIXED; 1706 1707 return rte_eth_linkstatus_set(dev, &link); 1708 } 1709 1710 static int 1711 vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 1712 { 1713 /* Link status doesn't change for stopped dev */ 1714 if (dev->data->dev_started == 0) 1715 return -1; 1716 1717 return __vmxnet3_dev_link_update(dev, wait_to_complete); 1718 } 1719 1720 /* Updating rxmode through Vmxnet3_DriverShared structure in adapter */ 1721 static void 1722 vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) 1723 { 1724 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1725 1726 if (set) 1727 rxConf->rxMode = rxConf->rxMode | feature; 1728 else 1729 rxConf->rxMode = rxConf->rxMode & (~feature); 1730 1731 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE); 1732 } 1733 1734 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1735 static int 1736 vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev) 1737 { 1738 struct vmxnet3_hw *hw = dev->data->dev_private; 1739 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1740 1741 memset(vf_table, 0, VMXNET3_VFT_TABLE_SIZE); 1742 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 1); 1743 1744 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1745 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1746 1747 return 0; 1748 } 1749 1750 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1751 static int 1752 vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev) 1753 { 1754 struct vmxnet3_hw *hw = dev->data->dev_private; 1755 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1756 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 1757 1758 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) 1759 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1760 else 1761 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1762 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0); 1763 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1764 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1765 1766 return 0; 1767 } 1768 1769 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1770 static int 1771 vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev) 1772 { 1773 struct vmxnet3_hw *hw = dev->data->dev_private; 1774 1775 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1); 1776 1777 return 0; 1778 } 1779 1780 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1781 static int 1782 vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev) 1783 { 1784 struct vmxnet3_hw *hw = dev->data->dev_private; 1785 1786 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 0); 1787 1788 return 0; 1789 } 1790 1791 /* Enable/disable filter on vlan */ 1792 static int 1793 vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vid, int on) 1794 { 1795 struct vmxnet3_hw *hw = dev->data->dev_private; 1796 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1797 uint32_t *vf_table = rxConf->vfTable; 1798 1799 /* save state for restore */ 1800 if (on) 1801 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1802 else 1803 VMXNET3_CLEAR_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1804 1805 /* don't change active filter if in promiscuous mode */ 1806 if (rxConf->rxMode & VMXNET3_RXM_PROMISC) 1807 return 0; 1808 1809 /* set in hardware */ 1810 if (on) 1811 VMXNET3_SET_VFTABLE_ENTRY(vf_table, vid); 1812 else 1813 VMXNET3_CLEAR_VFTABLE_ENTRY(vf_table, vid); 1814 1815 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1816 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1817 return 0; 1818 } 1819 1820 static int 1821 vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1822 { 1823 struct vmxnet3_hw *hw = dev->data->dev_private; 1824 Vmxnet3_DSDevRead *devRead = &hw->shared->devRead; 1825 uint32_t *vf_table = devRead->rxFilterConf.vfTable; 1826 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 1827 1828 if (mask & RTE_ETH_VLAN_STRIP_MASK) { 1829 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 1830 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 1831 else 1832 devRead->misc.uptFeatures &= ~UPT1_F_RXVLAN; 1833 1834 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1835 VMXNET3_CMD_UPDATE_FEATURE); 1836 } 1837 1838 if (mask & RTE_ETH_VLAN_FILTER_MASK) { 1839 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) 1840 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1841 else 1842 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1843 1844 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1845 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1846 } 1847 1848 return 0; 1849 } 1850 1851 static void 1852 vmxnet3_process_events(struct rte_eth_dev *dev) 1853 { 1854 struct vmxnet3_hw *hw = dev->data->dev_private; 1855 uint32_t events = hw->shared->ecr; 1856 int i; 1857 1858 if (!events) 1859 return; 1860 1861 /* 1862 * ECR bits when written with 1b are cleared. Hence write 1863 * events back to ECR so that the bits which were set will be reset. 1864 */ 1865 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events); 1866 1867 /* Check if link state has changed */ 1868 if (events & VMXNET3_ECR_LINK) { 1869 PMD_DRV_LOG(DEBUG, "Process events: VMXNET3_ECR_LINK event"); 1870 if (vmxnet3_dev_link_update(dev, 0) == 0) 1871 rte_eth_dev_callback_process(dev, 1872 RTE_ETH_EVENT_INTR_LSC, 1873 NULL); 1874 } 1875 1876 /* Check if there is an error on xmit/recv queues */ 1877 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 1878 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1879 VMXNET3_CMD_GET_QUEUE_STATUS); 1880 1881 PMD_DRV_LOG(ERR, "queue error event 0x%x for " 1882 RTE_ETHER_ADDR_PRT_FMT, events, 1883 hw->perm_addr[0], hw->perm_addr[1], 1884 hw->perm_addr[2], hw->perm_addr[3], 1885 hw->perm_addr[4], hw->perm_addr[5]); 1886 1887 for (i = 0; i < hw->num_tx_queues; i++) { 1888 if (hw->tqd_start[i].status.stopped) 1889 PMD_DRV_LOG(ERR, "tq %d error 0x%x", 1890 i, hw->tqd_start[i].status.error); 1891 } 1892 for (i = 0; i < hw->num_rx_queues; i++) { 1893 if (hw->rqd_start[i].status.stopped) 1894 PMD_DRV_LOG(ERR, "rq %d error 0x%x", 1895 i, hw->rqd_start[i].status.error); 1896 } 1897 1898 /* Have to reset the device */ 1899 /* Notify the application so that it can reset the device */ 1900 rte_eth_dev_callback_process(dev, 1901 RTE_ETH_EVENT_INTR_RESET, 1902 NULL); 1903 } 1904 1905 if (events & VMXNET3_ECR_DIC) 1906 PMD_DRV_LOG(DEBUG, "Device implementation change event."); 1907 1908 if (events & VMXNET3_ECR_DEBUG) 1909 PMD_DRV_LOG(DEBUG, "Debug event generated by device."); 1910 } 1911 1912 static void 1913 vmxnet3_interrupt_handler(void *param) 1914 { 1915 struct rte_eth_dev *dev = param; 1916 struct vmxnet3_hw *hw = dev->data->dev_private; 1917 uint32_t events; 1918 uint8 *eventIntrIdx; 1919 uint32 *intrCtrl; 1920 1921 PMD_INIT_FUNC_TRACE(); 1922 1923 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 1924 vmxnet3_disable_intr(hw, *eventIntrIdx); 1925 1926 events = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_ECR); 1927 if (events == 0) 1928 goto done; 1929 1930 PMD_DRV_LOG(DEBUG, "Reading events: 0x%X", events); 1931 vmxnet3_process_events(dev); 1932 done: 1933 vmxnet3_enable_intr(hw, *eventIntrIdx); 1934 } 1935 1936 static int 1937 vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1938 { 1939 struct vmxnet3_hw *hw = dev->data->dev_private; 1940 1941 vmxnet3_enable_intr(hw, 1942 rte_intr_vec_list_index_get(dev->intr_handle, 1943 queue_id)); 1944 1945 return 0; 1946 } 1947 1948 static int 1949 vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1950 { 1951 struct vmxnet3_hw *hw = dev->data->dev_private; 1952 1953 vmxnet3_disable_intr(hw, 1954 rte_intr_vec_list_index_get(dev->intr_handle, queue_id)); 1955 1956 return 0; 1957 } 1958 1959 RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd); 1960 RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map); 1961 RTE_PMD_REGISTER_KMOD_DEP(net_vmxnet3, "* igb_uio | uio_pci_generic | vfio-pci"); 1962 RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_init, init, NOTICE); 1963 RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_driver, driver, NOTICE); 1964 1965 static int 1966 vmxnet3_rss_reta_update(struct rte_eth_dev *dev, 1967 struct rte_eth_rss_reta_entry64 *reta_conf, 1968 uint16_t reta_size) 1969 { 1970 int i, idx, shift; 1971 struct vmxnet3_hw *hw = dev->data->dev_private; 1972 struct VMXNET3_RSSConf *dev_rss_conf = hw->rss_conf; 1973 1974 if (reta_size != dev_rss_conf->indTableSize) { 1975 PMD_DRV_LOG(ERR, 1976 "The size of hash lookup table configured (%d) doesn't match " 1977 "the supported number (%d)", 1978 reta_size, dev_rss_conf->indTableSize); 1979 return -EINVAL; 1980 } 1981 1982 for (i = 0; i < reta_size; i++) { 1983 idx = i / RTE_ETH_RETA_GROUP_SIZE; 1984 shift = i % RTE_ETH_RETA_GROUP_SIZE; 1985 if (reta_conf[idx].mask & RTE_BIT64(shift)) 1986 dev_rss_conf->indTable[i] = (uint8_t)reta_conf[idx].reta[shift]; 1987 } 1988 1989 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1990 VMXNET3_CMD_UPDATE_RSSIDT); 1991 1992 return 0; 1993 } 1994 1995 static int 1996 vmxnet3_rss_reta_query(struct rte_eth_dev *dev, 1997 struct rte_eth_rss_reta_entry64 *reta_conf, 1998 uint16_t reta_size) 1999 { 2000 int i, idx, shift; 2001 struct vmxnet3_hw *hw = dev->data->dev_private; 2002 struct VMXNET3_RSSConf *dev_rss_conf = hw->rss_conf; 2003 2004 if (reta_size != dev_rss_conf->indTableSize) { 2005 PMD_DRV_LOG(ERR, 2006 "Size of requested hash lookup table (%d) doesn't " 2007 "match the configured size (%d)", 2008 reta_size, dev_rss_conf->indTableSize); 2009 return -EINVAL; 2010 } 2011 2012 for (i = 0; i < reta_size; i++) { 2013 idx = i / RTE_ETH_RETA_GROUP_SIZE; 2014 shift = i % RTE_ETH_RETA_GROUP_SIZE; 2015 if (reta_conf[idx].mask & RTE_BIT64(shift)) 2016 reta_conf[idx].reta[shift] = dev_rss_conf->indTable[i]; 2017 } 2018 2019 return 0; 2020 } 2021