1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdalign.h> 7 #include <stdio.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <unistd.h> 12 #include <stdarg.h> 13 #include <fcntl.h> 14 #include <inttypes.h> 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 19 #include <rte_interrupts.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_pci.h> 23 #include <bus_pci_driver.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_memory.h> 26 #include <rte_memzone.h> 27 #include <rte_eal.h> 28 #include <rte_alarm.h> 29 #include <rte_ether.h> 30 #include <ethdev_driver.h> 31 #include <ethdev_pci.h> 32 #include <rte_string_fns.h> 33 #include <rte_malloc.h> 34 #include <dev_driver.h> 35 36 #include "base/vmxnet3_defs.h" 37 38 #include "vmxnet3_ring.h" 39 #include "vmxnet3_logs.h" 40 #include "vmxnet3_ethdev.h" 41 42 #define VMXNET3_TX_MAX_SEG UINT8_MAX 43 44 #define VMXNET3_TX_OFFLOAD_CAP \ 45 (RTE_ETH_TX_OFFLOAD_VLAN_INSERT | \ 46 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \ 47 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | \ 48 RTE_ETH_TX_OFFLOAD_TCP_TSO | \ 49 RTE_ETH_TX_OFFLOAD_MULTI_SEGS) 50 51 #define VMXNET3_RX_OFFLOAD_CAP \ 52 (RTE_ETH_RX_OFFLOAD_VLAN_STRIP | \ 53 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | \ 54 RTE_ETH_RX_OFFLOAD_SCATTER | \ 55 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | \ 56 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | \ 57 RTE_ETH_RX_OFFLOAD_TCP_LRO | \ 58 RTE_ETH_RX_OFFLOAD_RSS_HASH) 59 60 int vmxnet3_segs_dynfield_offset = -1; 61 62 static int eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev); 63 static int eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev); 64 static int vmxnet3_dev_configure(struct rte_eth_dev *dev); 65 static int vmxnet3_dev_start(struct rte_eth_dev *dev); 66 static int vmxnet3_dev_stop(struct rte_eth_dev *dev); 67 static int vmxnet3_dev_close(struct rte_eth_dev *dev); 68 static int vmxnet3_dev_reset(struct rte_eth_dev *dev); 69 static void vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set); 70 static int vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev); 71 static int vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev); 72 static int vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev); 73 static int vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev); 74 static int __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 75 int wait_to_complete); 76 static int vmxnet3_dev_link_update(struct rte_eth_dev *dev, 77 int wait_to_complete); 78 static void vmxnet3_hw_stats_save(struct vmxnet3_hw *hw); 79 static int vmxnet3_dev_stats_get(struct rte_eth_dev *dev, 80 struct rte_eth_stats *stats); 81 static int vmxnet3_dev_stats_reset(struct rte_eth_dev *dev); 82 static int vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 83 struct rte_eth_xstat_name *xstats, 84 unsigned int n); 85 static int vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, 86 struct rte_eth_xstat *xstats, unsigned int n); 87 static int vmxnet3_dev_info_get(struct rte_eth_dev *dev, 88 struct rte_eth_dev_info *dev_info); 89 static int vmxnet3_hw_ver_get(struct rte_eth_dev *dev, 90 char *fw_version, size_t fw_size); 91 static const uint32_t * 92 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev, 93 size_t *no_of_elements); 94 static int vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 95 static int vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, 96 uint16_t vid, int on); 97 static int vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 98 static int vmxnet3_mac_addr_set(struct rte_eth_dev *dev, 99 struct rte_ether_addr *mac_addr); 100 static void vmxnet3_process_events(struct rte_eth_dev *dev); 101 static void vmxnet3_interrupt_handler(void *param); 102 static int 103 vmxnet3_rss_reta_update(struct rte_eth_dev *dev, 104 struct rte_eth_rss_reta_entry64 *reta_conf, 105 uint16_t reta_size); 106 static int 107 vmxnet3_rss_reta_query(struct rte_eth_dev *dev, 108 struct rte_eth_rss_reta_entry64 *reta_conf, 109 uint16_t reta_size); 110 111 static int vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 112 uint16_t queue_id); 113 static int vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 114 uint16_t queue_id); 115 116 /* 117 * The set of PCI devices this driver supports 118 */ 119 #define VMWARE_PCI_VENDOR_ID 0x15AD 120 #define VMWARE_DEV_ID_VMXNET3 0x07B0 121 static const struct rte_pci_id pci_id_vmxnet3_map[] = { 122 { RTE_PCI_DEVICE(VMWARE_PCI_VENDOR_ID, VMWARE_DEV_ID_VMXNET3) }, 123 { .vendor_id = 0, /* sentinel */ }, 124 }; 125 126 static const struct eth_dev_ops vmxnet3_eth_dev_ops = { 127 .dev_configure = vmxnet3_dev_configure, 128 .dev_start = vmxnet3_dev_start, 129 .dev_stop = vmxnet3_dev_stop, 130 .dev_close = vmxnet3_dev_close, 131 .dev_reset = vmxnet3_dev_reset, 132 .link_update = vmxnet3_dev_link_update, 133 .promiscuous_enable = vmxnet3_dev_promiscuous_enable, 134 .promiscuous_disable = vmxnet3_dev_promiscuous_disable, 135 .allmulticast_enable = vmxnet3_dev_allmulticast_enable, 136 .allmulticast_disable = vmxnet3_dev_allmulticast_disable, 137 .mac_addr_set = vmxnet3_mac_addr_set, 138 .mtu_set = vmxnet3_dev_mtu_set, 139 .stats_get = vmxnet3_dev_stats_get, 140 .stats_reset = vmxnet3_dev_stats_reset, 141 .xstats_get = vmxnet3_dev_xstats_get, 142 .xstats_get_names = vmxnet3_dev_xstats_get_names, 143 .dev_infos_get = vmxnet3_dev_info_get, 144 .fw_version_get = vmxnet3_hw_ver_get, 145 .dev_supported_ptypes_get = vmxnet3_dev_supported_ptypes_get, 146 .vlan_filter_set = vmxnet3_dev_vlan_filter_set, 147 .vlan_offload_set = vmxnet3_dev_vlan_offload_set, 148 .rx_queue_setup = vmxnet3_dev_rx_queue_setup, 149 .rx_queue_release = vmxnet3_dev_rx_queue_release, 150 .rx_queue_intr_enable = vmxnet3_dev_rx_queue_intr_enable, 151 .rx_queue_intr_disable = vmxnet3_dev_rx_queue_intr_disable, 152 .tx_queue_setup = vmxnet3_dev_tx_queue_setup, 153 .tx_queue_release = vmxnet3_dev_tx_queue_release, 154 .reta_update = vmxnet3_rss_reta_update, 155 .reta_query = vmxnet3_rss_reta_query, 156 }; 157 158 struct vmxnet3_xstats_name_off { 159 char name[RTE_ETH_XSTATS_NAME_SIZE]; 160 unsigned int offset; 161 }; 162 163 /* tx_qX_ is prepended to the name string here */ 164 static const struct vmxnet3_xstats_name_off vmxnet3_txq_stat_strings[] = { 165 {"drop_total", offsetof(struct vmxnet3_txq_stats, drop_total)}, 166 {"drop_too_many_segs", offsetof(struct vmxnet3_txq_stats, drop_too_many_segs)}, 167 {"drop_tso", offsetof(struct vmxnet3_txq_stats, drop_tso)}, 168 {"tx_ring_full", offsetof(struct vmxnet3_txq_stats, tx_ring_full)}, 169 }; 170 171 /* rx_qX_ is prepended to the name string here */ 172 static const struct vmxnet3_xstats_name_off vmxnet3_rxq_stat_strings[] = { 173 {"drop_total", offsetof(struct vmxnet3_rxq_stats, drop_total)}, 174 {"drop_err", offsetof(struct vmxnet3_rxq_stats, drop_err)}, 175 {"drop_fcs", offsetof(struct vmxnet3_rxq_stats, drop_fcs)}, 176 {"rx_buf_alloc_failure", offsetof(struct vmxnet3_rxq_stats, rx_buf_alloc_failure)}, 177 }; 178 179 static const struct rte_memzone * 180 gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size, 181 const char *post_string, int socket_id, 182 uint16_t align, bool reuse) 183 { 184 char z_name[RTE_MEMZONE_NAMESIZE]; 185 const struct rte_memzone *mz; 186 187 snprintf(z_name, sizeof(z_name), "eth_p%d_%s", 188 dev->data->port_id, post_string); 189 190 mz = rte_memzone_lookup(z_name); 191 if (!reuse) { 192 rte_memzone_free(mz); 193 return rte_memzone_reserve_aligned(z_name, size, socket_id, 194 RTE_MEMZONE_IOVA_CONTIG, align); 195 } 196 197 if (mz) 198 return mz; 199 200 return rte_memzone_reserve_aligned(z_name, size, socket_id, 201 RTE_MEMZONE_IOVA_CONTIG, align); 202 } 203 204 /* 205 * Enable the given interrupt 206 */ 207 static void 208 vmxnet3_enable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx) 209 { 210 PMD_INIT_FUNC_TRACE(); 211 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 0); 212 } 213 214 /* 215 * Disable the given interrupt 216 */ 217 static void 218 vmxnet3_disable_intr(struct vmxnet3_hw *hw, unsigned int intr_idx) 219 { 220 PMD_INIT_FUNC_TRACE(); 221 VMXNET3_WRITE_BAR0_REG(hw, VMXNET3_REG_IMR + intr_idx * 8, 1); 222 } 223 224 /* 225 * Simple helper to get intrCtrl and eventIntrIdx based on config and hw version 226 */ 227 static void 228 vmxnet3_get_intr_ctrl_ev(struct vmxnet3_hw *hw, 229 uint8 **out_eventIntrIdx, 230 uint32 **out_intrCtrl) 231 { 232 233 if (VMXNET3_VERSION_GE_6(hw) && hw->queuesExtEnabled) { 234 *out_eventIntrIdx = &hw->shared->devReadExt.intrConfExt.eventIntrIdx; 235 *out_intrCtrl = &hw->shared->devReadExt.intrConfExt.intrCtrl; 236 } else { 237 *out_eventIntrIdx = &hw->shared->devRead.intrConf.eventIntrIdx; 238 *out_intrCtrl = &hw->shared->devRead.intrConf.intrCtrl; 239 } 240 } 241 242 /* 243 * Disable all intrs used by the device 244 */ 245 static void 246 vmxnet3_disable_all_intrs(struct vmxnet3_hw *hw) 247 { 248 int i; 249 uint8 *eventIntrIdx; 250 uint32 *intrCtrl; 251 252 PMD_INIT_FUNC_TRACE(); 253 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 254 255 *intrCtrl |= rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 256 257 for (i = 0; i < hw->intr.num_intrs; i++) 258 vmxnet3_disable_intr(hw, i); 259 } 260 261 #ifndef RTE_EXEC_ENV_FREEBSD 262 /* 263 * Enable all intrs used by the device 264 */ 265 static void 266 vmxnet3_enable_all_intrs(struct vmxnet3_hw *hw) 267 { 268 uint8 *eventIntrIdx; 269 uint32 *intrCtrl; 270 271 PMD_INIT_FUNC_TRACE(); 272 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 273 274 *intrCtrl &= rte_cpu_to_le_32(~VMXNET3_IC_DISABLE_ALL); 275 276 if (hw->intr.lsc_only) { 277 vmxnet3_enable_intr(hw, *eventIntrIdx); 278 } else { 279 int i; 280 281 for (i = 0; i < hw->intr.num_intrs; i++) 282 vmxnet3_enable_intr(hw, i); 283 } 284 } 285 #endif 286 287 /* 288 * Gets tx data ring descriptor size. 289 */ 290 static uint16_t 291 eth_vmxnet3_txdata_get(struct vmxnet3_hw *hw) 292 { 293 uint16 txdata_desc_size; 294 295 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 296 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 297 txdata_desc_size = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 298 299 return (txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE || 300 txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE || 301 txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK) ? 302 sizeof(struct Vmxnet3_TxDataDesc) : txdata_desc_size; 303 } 304 305 static int 306 eth_vmxnet3_setup_capabilities(struct vmxnet3_hw *hw, 307 struct rte_eth_dev *eth_dev) 308 { 309 uint32_t dcr, ptcr, value; 310 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 311 312 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 313 VMXNET3_CMD_GET_MAX_CAPABILITIES); 314 value = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 315 hw->max_capabilities[0] = value; 316 dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_DCR); 317 hw->DCR_capabilities[0] = dcr; 318 hw->used_DCR_capabilities[0] = 0; 319 ptcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_PTCR); 320 hw->PTCR_capabilities[0] = ptcr; 321 hw->used_PTCR_capabilities[0] = 0; 322 323 if (hw->uptv2_enabled && !(ptcr & (1 << VMXNET3_DCR_ERROR))) { 324 PMD_DRV_LOG(NOTICE, "UPTv2 enabled"); 325 hw->used_PTCR_capabilities[0] = ptcr; 326 } else { 327 /* Use all DCR capabilities, but disable large bar */ 328 hw->used_DCR_capabilities[0] = dcr & 329 (~(1UL << VMXNET3_CAP_LARGE_BAR)); 330 PMD_DRV_LOG(NOTICE, "UPTv2 disabled"); 331 } 332 if (hw->DCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 333 hw->PTCR_capabilities[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 334 if (hw->uptv2_enabled) { 335 hw->used_PTCR_capabilities[0] |= 336 (1UL << VMXNET3_CAP_OOORX_COMP); 337 } 338 } 339 if (hw->used_PTCR_capabilities[0]) { 340 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR, 341 hw->used_PTCR_capabilities[0]); 342 } else if (hw->used_DCR_capabilities[0]) { 343 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DCR, 344 hw->used_DCR_capabilities[0]); 345 } 346 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 347 dcr = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 348 hw->used_DCR_capabilities[0] = dcr; 349 PMD_DRV_LOG(DEBUG, "Dev " PCI_PRI_FMT ", vmxnet3 v%d, UPT enabled: %s, " 350 "DCR0=0x%08x, used DCR=0x%08x, " 351 "PTCR=0x%08x, used PTCR=0x%08x", 352 pci_dev->addr.domain, pci_dev->addr.bus, 353 pci_dev->addr.devid, pci_dev->addr.function, hw->version, 354 hw->uptv2_enabled ? "true" : "false", 355 hw->DCR_capabilities[0], hw->used_DCR_capabilities[0], 356 hw->PTCR_capabilities[0], hw->used_PTCR_capabilities[0]); 357 return 0; 358 } 359 360 /* 361 * It returns 0 on success. 362 */ 363 static int 364 eth_vmxnet3_dev_init(struct rte_eth_dev *eth_dev) 365 { 366 struct rte_pci_device *pci_dev; 367 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 368 uint32_t mac_hi, mac_lo, ver; 369 struct rte_eth_link link; 370 static const struct rte_mbuf_dynfield vmxnet3_segs_dynfield_desc = { 371 .name = VMXNET3_SEGS_DYNFIELD_NAME, 372 .size = sizeof(vmxnet3_segs_dynfield_t), 373 .align = alignof(vmxnet3_segs_dynfield_t), 374 }; 375 376 PMD_INIT_FUNC_TRACE(); 377 378 eth_dev->dev_ops = &vmxnet3_eth_dev_ops; 379 eth_dev->rx_pkt_burst = &vmxnet3_recv_pkts; 380 eth_dev->tx_pkt_burst = &vmxnet3_xmit_pkts; 381 eth_dev->tx_pkt_prepare = vmxnet3_prep_pkts; 382 eth_dev->rx_queue_count = vmxnet3_dev_rx_queue_count; 383 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 384 385 /* extra mbuf field is required to guess MSS */ 386 vmxnet3_segs_dynfield_offset = 387 rte_mbuf_dynfield_register(&vmxnet3_segs_dynfield_desc); 388 if (vmxnet3_segs_dynfield_offset < 0) { 389 PMD_INIT_LOG(ERR, "Cannot register mbuf field."); 390 return -rte_errno; 391 } 392 393 /* 394 * for secondary processes, we don't initialize any further as primary 395 * has already done this work. 396 */ 397 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 398 return 0; 399 400 rte_eth_copy_pci_info(eth_dev, pci_dev); 401 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 402 403 /* Vendor and Device ID need to be set before init of shared code */ 404 hw->device_id = pci_dev->id.device_id; 405 hw->vendor_id = pci_dev->id.vendor_id; 406 hw->adapter_stopped = TRUE; 407 hw->hw_addr0 = (void *)pci_dev->mem_resource[0].addr; 408 hw->hw_addr1 = (void *)pci_dev->mem_resource[1].addr; 409 410 hw->num_rx_queues = 1; 411 hw->num_tx_queues = 1; 412 hw->bufs_per_pkt = 1; 413 414 /* Check h/w version compatibility with driver. */ 415 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_VRRS); 416 417 if (ver & (1 << VMXNET3_REV_7)) { 418 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 419 1 << VMXNET3_REV_7); 420 hw->version = VMXNET3_REV_7 + 1; 421 } else if (ver & (1 << VMXNET3_REV_6)) { 422 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 423 1 << VMXNET3_REV_6); 424 hw->version = VMXNET3_REV_6 + 1; 425 } else if (ver & (1 << VMXNET3_REV_5)) { 426 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 427 1 << VMXNET3_REV_5); 428 hw->version = VMXNET3_REV_5 + 1; 429 } else if (ver & (1 << VMXNET3_REV_4)) { 430 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 431 1 << VMXNET3_REV_4); 432 hw->version = VMXNET3_REV_4 + 1; 433 } else if (ver & (1 << VMXNET3_REV_3)) { 434 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 435 1 << VMXNET3_REV_3); 436 hw->version = VMXNET3_REV_3 + 1; 437 } else if (ver & (1 << VMXNET3_REV_2)) { 438 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 439 1 << VMXNET3_REV_2); 440 hw->version = VMXNET3_REV_2 + 1; 441 } else if (ver & (1 << VMXNET3_REV_1)) { 442 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_VRRS, 443 1 << VMXNET3_REV_1); 444 hw->version = VMXNET3_REV_1 + 1; 445 } else { 446 PMD_INIT_LOG(ERR, "Incompatible hardware version: %d", ver); 447 return -EIO; 448 } 449 450 PMD_INIT_LOG(INFO, "Using device v%d", hw->version); 451 452 /* Check UPT version compatibility with driver. */ 453 ver = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_UVRS); 454 PMD_INIT_LOG(DEBUG, "UPT hardware version : %d", ver); 455 if (ver & 0x1) 456 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_UVRS, 1); 457 else { 458 PMD_INIT_LOG(ERR, "Incompatible UPT version."); 459 return -EIO; 460 } 461 462 if (VMXNET3_VERSION_GE_7(hw)) { 463 /* start with UPTv2 enabled to avoid ESXi issues */ 464 hw->uptv2_enabled = TRUE; 465 eth_vmxnet3_setup_capabilities(hw, eth_dev); 466 } 467 468 if (hw->used_DCR_capabilities[0] & (1 << VMXNET3_CAP_LARGE_BAR)) { 469 hw->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 470 hw->rx_prod_offset[0] = VMXNET3_REG_LB_RXPROD; 471 hw->rx_prod_offset[1] = VMXNET3_REG_LB_RXPROD2; 472 } else { 473 hw->tx_prod_offset = VMXNET3_REG_TXPROD; 474 hw->rx_prod_offset[0] = VMXNET3_REG_RXPROD; 475 hw->rx_prod_offset[1] = VMXNET3_REG_RXPROD2; 476 } 477 478 /* Getting MAC Address */ 479 mac_lo = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACL); 480 mac_hi = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_MACH); 481 memcpy(hw->perm_addr, &mac_lo, 4); 482 memcpy(hw->perm_addr + 4, &mac_hi, 2); 483 484 /* Allocate memory for storing MAC addresses */ 485 eth_dev->data->mac_addrs = rte_zmalloc("vmxnet3", RTE_ETHER_ADDR_LEN * 486 VMXNET3_MAX_MAC_ADDRS, 0); 487 if (eth_dev->data->mac_addrs == NULL) { 488 PMD_INIT_LOG(ERR, 489 "Failed to allocate %d bytes needed to store MAC addresses", 490 RTE_ETHER_ADDR_LEN * VMXNET3_MAX_MAC_ADDRS); 491 return -ENOMEM; 492 } 493 /* Copy the permanent MAC address */ 494 rte_ether_addr_copy((struct rte_ether_addr *)hw->perm_addr, 495 ð_dev->data->mac_addrs[0]); 496 497 PMD_INIT_LOG(DEBUG, "MAC Address : " RTE_ETHER_ADDR_PRT_FMT, 498 hw->perm_addr[0], hw->perm_addr[1], hw->perm_addr[2], 499 hw->perm_addr[3], hw->perm_addr[4], hw->perm_addr[5]); 500 501 /* Put device in Quiesce Mode */ 502 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 503 504 /* allow untagged pkts */ 505 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, 0); 506 507 hw->txdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 508 eth_vmxnet3_txdata_get(hw) : sizeof(struct Vmxnet3_TxDataDesc); 509 510 hw->rxdata_desc_size = VMXNET3_VERSION_GE_3(hw) ? 511 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 512 RTE_ASSERT((hw->rxdata_desc_size & ~VMXNET3_RXDATA_DESC_SIZE_MASK) == 513 hw->rxdata_desc_size); 514 515 /* clear shadow stats */ 516 memset(hw->saved_tx_stats, 0, sizeof(hw->saved_tx_stats)); 517 memset(hw->saved_rx_stats, 0, sizeof(hw->saved_rx_stats)); 518 519 /* clear snapshot stats */ 520 memset(hw->snapshot_tx_stats, 0, sizeof(hw->snapshot_tx_stats)); 521 memset(hw->snapshot_rx_stats, 0, sizeof(hw->snapshot_rx_stats)); 522 523 /* set the initial link status */ 524 memset(&link, 0, sizeof(link)); 525 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 526 link.link_speed = RTE_ETH_SPEED_NUM_10G; 527 link.link_autoneg = RTE_ETH_LINK_FIXED; 528 rte_eth_linkstatus_set(eth_dev, &link); 529 530 return 0; 531 } 532 533 static int 534 eth_vmxnet3_dev_uninit(struct rte_eth_dev *eth_dev) 535 { 536 struct vmxnet3_hw *hw = eth_dev->data->dev_private; 537 538 PMD_INIT_FUNC_TRACE(); 539 540 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 541 return 0; 542 543 if (hw->adapter_stopped == 0) { 544 PMD_INIT_LOG(DEBUG, "Device has not been closed."); 545 return -EBUSY; 546 } 547 548 return 0; 549 } 550 551 static int eth_vmxnet3_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 552 struct rte_pci_device *pci_dev) 553 { 554 return rte_eth_dev_pci_generic_probe(pci_dev, 555 sizeof(struct vmxnet3_hw), eth_vmxnet3_dev_init); 556 } 557 558 static int eth_vmxnet3_pci_remove(struct rte_pci_device *pci_dev) 559 { 560 return rte_eth_dev_pci_generic_remove(pci_dev, eth_vmxnet3_dev_uninit); 561 } 562 563 static struct rte_pci_driver rte_vmxnet3_pmd = { 564 .id_table = pci_id_vmxnet3_map, 565 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 566 .probe = eth_vmxnet3_pci_probe, 567 .remove = eth_vmxnet3_pci_remove, 568 }; 569 570 static void 571 vmxnet3_alloc_intr_resources(struct rte_eth_dev *dev) 572 { 573 struct vmxnet3_hw *hw = dev->data->dev_private; 574 uint32_t cfg; 575 int nvec = 1; /* for link event */ 576 577 /* intr settings */ 578 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 579 VMXNET3_CMD_GET_CONF_INTR); 580 cfg = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 581 hw->intr.type = cfg & 0x3; 582 hw->intr.mask_mode = (cfg >> 2) & 0x3; 583 584 if (hw->intr.type == VMXNET3_IT_AUTO) 585 hw->intr.type = VMXNET3_IT_MSIX; 586 587 if (hw->intr.type == VMXNET3_IT_MSIX) { 588 /* only support shared tx/rx intr */ 589 if (hw->num_tx_queues != hw->num_rx_queues) 590 goto msix_err; 591 592 nvec += hw->num_rx_queues; 593 hw->intr.num_intrs = nvec; 594 return; 595 } 596 597 msix_err: 598 /* the tx/rx queue interrupt will be disabled */ 599 hw->intr.num_intrs = 2; 600 hw->intr.lsc_only = TRUE; 601 PMD_INIT_LOG(INFO, "Enabled MSI-X with %d vectors", hw->intr.num_intrs); 602 } 603 604 static int 605 vmxnet3_dev_configure(struct rte_eth_dev *dev) 606 { 607 const struct rte_memzone *mz; 608 struct vmxnet3_hw *hw = dev->data->dev_private; 609 size_t size; 610 611 PMD_INIT_FUNC_TRACE(); 612 613 if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) 614 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; 615 616 if (!VMXNET3_VERSION_GE_6(hw)) { 617 if (!rte_is_power_of_2(dev->data->nb_rx_queues)) { 618 PMD_INIT_LOG(ERR, 619 "ERROR: Number of rx queues not power of 2"); 620 return -EINVAL; 621 } 622 } 623 624 /* At this point, the number of queues requested has already 625 * been validated against dev_infos max queues by EAL 626 */ 627 if (dev->data->nb_rx_queues > VMXNET3_MAX_RX_QUEUES || 628 dev->data->nb_tx_queues > VMXNET3_MAX_TX_QUEUES) { 629 hw->queuesExtEnabled = 1; 630 } else { 631 hw->queuesExtEnabled = 0; 632 } 633 634 size = dev->data->nb_rx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 635 dev->data->nb_tx_queues * sizeof(struct Vmxnet3_RxQueueDesc); 636 637 if (size > UINT16_MAX) 638 return -EINVAL; 639 640 hw->num_rx_queues = (uint8_t)dev->data->nb_rx_queues; 641 hw->num_tx_queues = (uint8_t)dev->data->nb_tx_queues; 642 643 /* 644 * Allocate a memzone for Vmxnet3_DriverShared - Vmxnet3_DSDevRead 645 * on current socket 646 */ 647 mz = gpa_zone_reserve(dev, sizeof(struct Vmxnet3_DriverShared), 648 "shared", rte_socket_id(), 8, 1); 649 650 if (mz == NULL) { 651 PMD_INIT_LOG(ERR, "ERROR: Creating shared zone"); 652 return -ENOMEM; 653 } 654 memset(mz->addr, 0, mz->len); 655 656 hw->shared = mz->addr; 657 hw->sharedPA = mz->iova; 658 659 /* 660 * Allocate a memzone for Vmxnet3_RxQueueDesc - Vmxnet3_TxQueueDesc 661 * on current socket. 662 * 663 * We cannot reuse this memzone from previous allocation as its size 664 * depends on the number of tx and rx queues, which could be different 665 * from one config to another. 666 */ 667 mz = gpa_zone_reserve(dev, size, "queuedesc", rte_socket_id(), 668 VMXNET3_QUEUE_DESC_ALIGN, 0); 669 if (mz == NULL) { 670 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); 671 return -ENOMEM; 672 } 673 memset(mz->addr, 0, mz->len); 674 675 hw->tqd_start = (Vmxnet3_TxQueueDesc *)mz->addr; 676 hw->rqd_start = (Vmxnet3_RxQueueDesc *)(hw->tqd_start + hw->num_tx_queues); 677 678 hw->queueDescPA = mz->iova; 679 hw->queue_desc_len = (uint16_t)size; 680 681 if (dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 682 /* Allocate memory structure for UPT1_RSSConf and configure */ 683 mz = gpa_zone_reserve(dev, sizeof(struct VMXNET3_RSSConf), 684 "rss_conf", rte_socket_id(), 685 RTE_CACHE_LINE_SIZE, 1); 686 if (mz == NULL) { 687 PMD_INIT_LOG(ERR, 688 "ERROR: Creating rss_conf structure zone"); 689 return -ENOMEM; 690 } 691 memset(mz->addr, 0, mz->len); 692 693 hw->rss_conf = mz->addr; 694 hw->rss_confPA = mz->iova; 695 } 696 697 vmxnet3_alloc_intr_resources(dev); 698 699 return 0; 700 } 701 702 static void 703 vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr) 704 { 705 uint32_t val; 706 707 PMD_INIT_LOG(DEBUG, 708 "Writing MAC Address : " RTE_ETHER_ADDR_PRT_FMT, 709 addr[0], addr[1], addr[2], 710 addr[3], addr[4], addr[5]); 711 712 memcpy(&val, addr, 4); 713 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACL, val); 714 715 memcpy(&val, addr + 4, 2); 716 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val); 717 } 718 719 /* 720 * Configure the hardware to generate MSI-X interrupts. 721 * If setting up MSIx fails, try setting up MSI (only 1 interrupt vector 722 * which will be disabled to allow lsc to work). 723 * 724 * Returns 0 on success and -1 otherwise. 725 */ 726 static int 727 vmxnet3_configure_msix(struct rte_eth_dev *dev) 728 { 729 struct vmxnet3_hw *hw = dev->data->dev_private; 730 struct rte_intr_handle *intr_handle = dev->intr_handle; 731 uint16_t intr_vector; 732 int i; 733 734 hw->intr.event_intr_idx = 0; 735 736 /* only vfio-pci driver can support interrupt mode. */ 737 if (!rte_intr_cap_multiple(intr_handle) || 738 dev->data->dev_conf.intr_conf.rxq == 0) 739 return -1; 740 741 intr_vector = dev->data->nb_rx_queues; 742 if (intr_vector > MAX_RX_QUEUES(hw)) { 743 PMD_INIT_LOG(ERR, "At most %d intr queues supported", 744 MAX_RX_QUEUES(hw)); 745 return -ENOTSUP; 746 } 747 748 if (rte_intr_efd_enable(intr_handle, intr_vector)) { 749 PMD_INIT_LOG(ERR, "Failed to enable fastpath event fd"); 750 return -1; 751 } 752 753 if (rte_intr_dp_is_en(intr_handle)) { 754 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec", 755 dev->data->nb_rx_queues)) { 756 PMD_INIT_LOG(ERR, "Failed to allocate %d Rx queues intr_vec", 757 dev->data->nb_rx_queues); 758 rte_intr_efd_disable(intr_handle); 759 return -ENOMEM; 760 } 761 } 762 763 if (!rte_intr_allow_others(intr_handle) && 764 dev->data->dev_conf.intr_conf.lsc != 0) { 765 PMD_INIT_LOG(ERR, "not enough intr vector to support both Rx interrupt and LSC"); 766 rte_intr_vec_list_free(intr_handle); 767 rte_intr_efd_disable(intr_handle); 768 return -1; 769 } 770 771 /* if we cannot allocate one MSI-X vector per queue, don't enable 772 * interrupt mode. 773 */ 774 if (hw->intr.num_intrs != 775 (rte_intr_nb_efd_get(intr_handle) + 1)) { 776 PMD_INIT_LOG(ERR, "Device configured with %d Rx intr vectors, expecting %d", 777 hw->intr.num_intrs, 778 rte_intr_nb_efd_get(intr_handle) + 1); 779 rte_intr_vec_list_free(intr_handle); 780 rte_intr_efd_disable(intr_handle); 781 return -1; 782 } 783 784 for (i = 0; i < dev->data->nb_rx_queues; i++) 785 if (rte_intr_vec_list_index_set(intr_handle, i, i + 1)) 786 return -rte_errno; 787 788 for (i = 0; i < hw->intr.num_intrs; i++) 789 hw->intr.mod_levels[i] = UPT1_IML_ADAPTIVE; 790 791 PMD_INIT_LOG(INFO, "intr type %u, mode %u, %u vectors allocated", 792 hw->intr.type, hw->intr.mask_mode, hw->intr.num_intrs); 793 794 return 0; 795 } 796 797 static int 798 vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev) 799 { 800 struct vmxnet3_hw *hw = dev->data->dev_private; 801 Vmxnet3_DriverShared *shared = hw->shared; 802 Vmxnet3_CmdInfo *cmdInfo; 803 struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES]; 804 uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES]; 805 uint32_t num, i, j, size; 806 807 if (hw->memRegsPA == 0) { 808 const struct rte_memzone *mz; 809 810 size = sizeof(Vmxnet3_MemRegs) + 811 (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) * 812 sizeof(Vmxnet3_MemoryRegion); 813 814 mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8, 815 1); 816 if (mz == NULL) { 817 PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone"); 818 return -ENOMEM; 819 } 820 memset(mz->addr, 0, mz->len); 821 hw->memRegs = mz->addr; 822 hw->memRegsPA = mz->iova; 823 } 824 825 num = hw->num_rx_queues; 826 827 for (i = 0; i < num; i++) { 828 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 829 830 mp[i] = rxq->mp; 831 index[i] = 1 << i; 832 } 833 834 /* 835 * The same mempool could be used by multiple queues. In such a case, 836 * remove duplicate mempool entries. Only one entry is kept with 837 * bitmask indicating queues that are using this mempool. 838 */ 839 for (i = 1; i < num; i++) { 840 for (j = 0; j < i; j++) { 841 if (mp[i] == mp[j]) { 842 mp[i] = NULL; 843 index[j] |= 1 << i; 844 break; 845 } 846 } 847 } 848 849 j = 0; 850 for (i = 0; i < num; i++) { 851 if (mp[i] == NULL) 852 continue; 853 854 Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j]; 855 856 mr->startPA = 857 (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->iova; 858 mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ? 859 STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX; 860 mr->txQueueBits = index[i]; 861 mr->rxQueueBits = index[i]; 862 863 PMD_INIT_LOG(INFO, 864 "index: %u startPA: %" PRIu64 " length: %u, " 865 "rxBits: %x", 866 j, mr->startPA, mr->length, mr->rxQueueBits); 867 j++; 868 } 869 hw->memRegs->numRegs = j; 870 PMD_INIT_LOG(INFO, "numRegs: %u", j); 871 872 size = sizeof(Vmxnet3_MemRegs) + 873 (j - 1) * sizeof(Vmxnet3_MemoryRegion); 874 875 cmdInfo = &shared->cu.cmdInfo; 876 cmdInfo->varConf.confVer = 1; 877 cmdInfo->varConf.confLen = size; 878 cmdInfo->varConf.confPA = hw->memRegsPA; 879 880 return 0; 881 } 882 883 static int 884 vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) 885 { 886 struct rte_eth_conf port_conf = dev->data->dev_conf; 887 struct vmxnet3_hw *hw = dev->data->dev_private; 888 struct rte_intr_handle *intr_handle = dev->intr_handle; 889 uint32_t mtu = dev->data->mtu; 890 Vmxnet3_DriverShared *shared = hw->shared; 891 Vmxnet3_DSDevRead *devRead = &shared->devRead; 892 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 893 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 894 uint32_t i; 895 int ret; 896 897 hw->mtu = mtu; 898 899 shared->magic = VMXNET3_REV1_MAGIC; 900 devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM; 901 902 /* Setting up Guest OS information */ 903 devRead->misc.driverInfo.gos.gosBits = sizeof(void *) == 4 ? 904 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64; 905 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 906 devRead->misc.driverInfo.vmxnet3RevSpt = 1; 907 devRead->misc.driverInfo.uptVerSpt = 1; 908 909 devRead->misc.mtu = rte_le_to_cpu_32(mtu); 910 devRead->misc.queueDescPA = hw->queueDescPA; 911 devRead->misc.queueDescLen = hw->queue_desc_len; 912 devRead->misc.numTxQueues = hw->num_tx_queues; 913 devRead->misc.numRxQueues = hw->num_rx_queues; 914 915 for (i = 0; i < hw->num_tx_queues; i++) { 916 Vmxnet3_TxQueueDesc *tqd = &hw->tqd_start[i]; 917 vmxnet3_tx_queue_t *txq = dev->data->tx_queues[i]; 918 919 txq->shared = &hw->tqd_start[i]; 920 921 tqd->ctrl.txNumDeferred = 0; 922 tqd->ctrl.txThreshold = 1; 923 tqd->conf.txRingBasePA = txq->cmd_ring.basePA; 924 tqd->conf.compRingBasePA = txq->comp_ring.basePA; 925 tqd->conf.dataRingBasePA = txq->data_ring.basePA; 926 927 tqd->conf.txRingSize = txq->cmd_ring.size; 928 tqd->conf.compRingSize = txq->comp_ring.size; 929 tqd->conf.dataRingSize = txq->data_ring.size; 930 tqd->conf.txDataRingDescSize = txq->txdata_desc_size; 931 932 if (hw->intr.lsc_only) 933 tqd->conf.intrIdx = 1; 934 else 935 tqd->conf.intrIdx = 936 rte_intr_vec_list_index_get(intr_handle, 937 i); 938 tqd->status.stopped = TRUE; 939 tqd->status.error = 0; 940 memset(&tqd->stats, 0, sizeof(tqd->stats)); 941 } 942 943 for (i = 0; i < hw->num_rx_queues; i++) { 944 Vmxnet3_RxQueueDesc *rqd = &hw->rqd_start[i]; 945 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; 946 947 rxq->shared = &hw->rqd_start[i]; 948 949 rqd->conf.rxRingBasePA[0] = rxq->cmd_ring[0].basePA; 950 rqd->conf.rxRingBasePA[1] = rxq->cmd_ring[1].basePA; 951 rqd->conf.compRingBasePA = rxq->comp_ring.basePA; 952 953 rqd->conf.rxRingSize[0] = rxq->cmd_ring[0].size; 954 rqd->conf.rxRingSize[1] = rxq->cmd_ring[1].size; 955 rqd->conf.compRingSize = rxq->comp_ring.size; 956 957 if (VMXNET3_VERSION_GE_3(hw)) { 958 rqd->conf.rxDataRingBasePA = rxq->data_ring.basePA; 959 rqd->conf.rxDataRingDescSize = rxq->data_desc_size; 960 } 961 962 if (hw->intr.lsc_only) 963 rqd->conf.intrIdx = 1; 964 else 965 rqd->conf.intrIdx = 966 rte_intr_vec_list_index_get(intr_handle, 967 i); 968 rqd->status.stopped = TRUE; 969 rqd->status.error = 0; 970 memset(&rqd->stats, 0, sizeof(rqd->stats)); 971 } 972 973 /* intr settings */ 974 if (VMXNET3_VERSION_GE_6(hw) && hw->queuesExtEnabled) { 975 devReadExt->intrConfExt.autoMask = hw->intr.mask_mode == 976 VMXNET3_IMM_AUTO; 977 devReadExt->intrConfExt.numIntrs = hw->intr.num_intrs; 978 for (i = 0; i < hw->intr.num_intrs; i++) 979 devReadExt->intrConfExt.modLevels[i] = 980 hw->intr.mod_levels[i]; 981 982 devReadExt->intrConfExt.eventIntrIdx = hw->intr.event_intr_idx; 983 devReadExt->intrConfExt.intrCtrl |= 984 rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 985 } else { 986 devRead->intrConf.autoMask = hw->intr.mask_mode == 987 VMXNET3_IMM_AUTO; 988 devRead->intrConf.numIntrs = hw->intr.num_intrs; 989 for (i = 0; i < hw->intr.num_intrs; i++) 990 devRead->intrConf.modLevels[i] = hw->intr.mod_levels[i]; 991 992 devRead->intrConf.eventIntrIdx = hw->intr.event_intr_idx; 993 devRead->intrConf.intrCtrl |= rte_cpu_to_le_32(VMXNET3_IC_DISABLE_ALL); 994 } 995 996 /* RxMode set to 0 of VMXNET3_RXM_xxx */ 997 devRead->rxFilterConf.rxMode = 0; 998 999 /* Setting up feature flags */ 1000 if (rx_offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 1001 devRead->misc.uptFeatures |= VMXNET3_F_RXCSUM; 1002 1003 if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) { 1004 devRead->misc.uptFeatures |= VMXNET3_F_LRO; 1005 devRead->misc.maxNumRxSG = 0; 1006 } 1007 1008 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 1009 ret = vmxnet3_rss_configure(dev); 1010 if (ret != VMXNET3_SUCCESS) 1011 return ret; 1012 1013 devRead->misc.uptFeatures |= VMXNET3_F_RSS; 1014 devRead->rssConfDesc.confVer = 1; 1015 devRead->rssConfDesc.confLen = sizeof(struct VMXNET3_RSSConf); 1016 devRead->rssConfDesc.confPA = hw->rss_confPA; 1017 } 1018 1019 ret = vmxnet3_dev_vlan_offload_set(dev, 1020 RTE_ETH_VLAN_STRIP_MASK | RTE_ETH_VLAN_FILTER_MASK); 1021 if (ret) 1022 return ret; 1023 1024 vmxnet3_write_mac(hw, dev->data->mac_addrs->addr_bytes); 1025 1026 return VMXNET3_SUCCESS; 1027 } 1028 1029 static void 1030 vmxnet3_init_bufsize(struct vmxnet3_hw *hw) 1031 { 1032 struct Vmxnet3_DriverShared *shared = hw->shared; 1033 union Vmxnet3_CmdInfo *cmd_info = &shared->cu.cmdInfo; 1034 1035 if (!VMXNET3_VERSION_GE_7(hw)) 1036 return; 1037 1038 cmd_info->ringBufSize.ring1BufSizeType0 = hw->rxdata_buf_size; 1039 cmd_info->ringBufSize.ring1BufSizeType1 = 0; 1040 cmd_info->ringBufSize.ring2BufSizeType1 = hw->rxdata_buf_size; 1041 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1042 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 1043 } 1044 1045 /* 1046 * Configure device link speed and setup link. 1047 * Must be called after eth_vmxnet3_dev_init. Other wise it might fail 1048 * It returns 0 on success. 1049 */ 1050 static int 1051 vmxnet3_dev_start(struct rte_eth_dev *dev) 1052 { 1053 int ret; 1054 struct vmxnet3_hw *hw = dev->data->dev_private; 1055 uint16_t i; 1056 1057 PMD_INIT_FUNC_TRACE(); 1058 1059 /* Save stats before it is reset by CMD_ACTIVATE */ 1060 vmxnet3_hw_stats_save(hw); 1061 1062 /* configure MSI-X */ 1063 ret = vmxnet3_configure_msix(dev); 1064 if (ret < 0) { 1065 /* revert to lsc only */ 1066 hw->intr.num_intrs = 2; 1067 hw->intr.lsc_only = TRUE; 1068 } 1069 1070 ret = vmxnet3_setup_driver_shared(dev); 1071 if (ret != VMXNET3_SUCCESS) 1072 return ret; 1073 1074 /* Exchange shared data with device */ 1075 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 1076 VMXNET3_GET_ADDR_LO(hw->sharedPA)); 1077 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 1078 VMXNET3_GET_ADDR_HI(hw->sharedPA)); 1079 1080 /* Activate device by register write */ 1081 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV); 1082 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1083 1084 if (ret != 0) { 1085 PMD_INIT_LOG(ERR, "Device activation: UNSUCCESSFUL"); 1086 return -EINVAL; 1087 } 1088 1089 /* Check memregs restrictions first */ 1090 if (dev->data->nb_rx_queues <= VMXNET3_MAX_RX_QUEUES && 1091 dev->data->nb_tx_queues <= VMXNET3_MAX_TX_QUEUES) { 1092 ret = vmxnet3_dev_setup_memreg(dev); 1093 if (ret == 0) { 1094 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1095 VMXNET3_CMD_REGISTER_MEMREGS); 1096 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1097 if (ret != 0) 1098 PMD_INIT_LOG(DEBUG, 1099 "Failed in setup memory region cmd"); 1100 ret = 0; 1101 } else { 1102 PMD_INIT_LOG(DEBUG, "Failed to setup memory region"); 1103 } 1104 } else { 1105 PMD_INIT_LOG(WARNING, "Memregs can't init (rx: %d, tx: %d)", 1106 dev->data->nb_rx_queues, dev->data->nb_tx_queues); 1107 } 1108 1109 if (VMXNET3_VERSION_GE_4(hw) && 1110 dev->data->dev_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_RSS) { 1111 /* Check for additional RSS */ 1112 ret = vmxnet3_v4_rss_configure(dev); 1113 if (ret != VMXNET3_SUCCESS) { 1114 PMD_INIT_LOG(ERR, "Failed to configure v4 RSS"); 1115 return ret; 1116 } 1117 } 1118 1119 /* 1120 * Load RX queues with blank mbufs and update next2fill index for device 1121 * Update RxMode of the device 1122 */ 1123 ret = vmxnet3_dev_rxtx_init(dev); 1124 if (ret != VMXNET3_SUCCESS) { 1125 PMD_INIT_LOG(ERR, "Device queue init: UNSUCCESSFUL"); 1126 return ret; 1127 } 1128 1129 vmxnet3_init_bufsize(hw); 1130 1131 hw->adapter_stopped = FALSE; 1132 1133 /* Setting proper Rx Mode and issue Rx Mode Update command */ 1134 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); 1135 1136 #ifndef RTE_EXEC_ENV_FREEBSD 1137 /* Setup interrupt callback */ 1138 rte_intr_callback_register(dev->intr_handle, 1139 vmxnet3_interrupt_handler, dev); 1140 1141 if (rte_intr_enable(dev->intr_handle) < 0) { 1142 PMD_INIT_LOG(ERR, "interrupt enable failed"); 1143 return -EIO; 1144 } 1145 1146 /* enable all intrs */ 1147 vmxnet3_enable_all_intrs(hw); 1148 #endif 1149 1150 vmxnet3_process_events(dev); 1151 1152 /* 1153 * Update link state from device since this won't be 1154 * done upon starting with lsc in use. This is done 1155 * only after enabling interrupts to avoid any race 1156 * where the link state could change without an 1157 * interrupt being fired. 1158 */ 1159 __vmxnet3_dev_link_update(dev, 0); 1160 1161 for (i = 0; i < dev->data->nb_rx_queues; i++) 1162 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 1163 for (i = 0; i < dev->data->nb_tx_queues; i++) 1164 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; 1165 1166 return VMXNET3_SUCCESS; 1167 } 1168 1169 /* 1170 * Stop device: disable rx and tx functions to allow for reconfiguring. 1171 */ 1172 static int 1173 vmxnet3_dev_stop(struct rte_eth_dev *dev) 1174 { 1175 struct rte_eth_link link; 1176 struct vmxnet3_hw *hw = dev->data->dev_private; 1177 struct rte_intr_handle *intr_handle = dev->intr_handle; 1178 uint16_t i; 1179 int ret; 1180 1181 PMD_INIT_FUNC_TRACE(); 1182 1183 if (hw->adapter_stopped == 1) { 1184 PMD_INIT_LOG(DEBUG, "Device already stopped."); 1185 return 0; 1186 } 1187 1188 do { 1189 /* Unregister has lock to make sure there is no running cb. 1190 * This has to happen first since vmxnet3_interrupt_handler 1191 * reenables interrupts by calling vmxnet3_enable_intr 1192 */ 1193 ret = rte_intr_callback_unregister(intr_handle, 1194 vmxnet3_interrupt_handler, 1195 (void *)-1); 1196 } while (ret == -EAGAIN); 1197 1198 if (ret < 0) 1199 PMD_DRV_LOG(ERR, "Error attempting to unregister intr cb: %d", 1200 ret); 1201 1202 PMD_INIT_LOG(DEBUG, "Disabled %d intr callbacks", ret); 1203 1204 /* disable interrupts */ 1205 vmxnet3_disable_all_intrs(hw); 1206 1207 rte_intr_disable(intr_handle); 1208 1209 /* Clean datapath event and queue/vector mapping */ 1210 rte_intr_efd_disable(intr_handle); 1211 rte_intr_vec_list_free(intr_handle); 1212 1213 /* quiesce the device first */ 1214 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV); 1215 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAL, 0); 1216 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_DSAH, 0); 1217 1218 /* reset the device */ 1219 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 1220 PMD_INIT_LOG(DEBUG, "Device reset."); 1221 1222 vmxnet3_dev_clear_queues(dev); 1223 1224 /* Clear recorded link status */ 1225 memset(&link, 0, sizeof(link)); 1226 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 1227 link.link_speed = RTE_ETH_SPEED_NUM_10G; 1228 link.link_autoneg = RTE_ETH_LINK_FIXED; 1229 rte_eth_linkstatus_set(dev, &link); 1230 1231 hw->adapter_stopped = 1; 1232 dev->data->dev_started = 0; 1233 1234 for (i = 0; i < dev->data->nb_rx_queues; i++) 1235 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1236 for (i = 0; i < dev->data->nb_tx_queues; i++) 1237 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; 1238 1239 return 0; 1240 } 1241 1242 static void 1243 vmxnet3_free_queues(struct rte_eth_dev *dev) 1244 { 1245 int i; 1246 1247 PMD_INIT_FUNC_TRACE(); 1248 1249 for (i = 0; i < dev->data->nb_rx_queues; i++) 1250 vmxnet3_dev_rx_queue_release(dev, i); 1251 dev->data->nb_rx_queues = 0; 1252 1253 for (i = 0; i < dev->data->nb_tx_queues; i++) 1254 vmxnet3_dev_tx_queue_release(dev, i); 1255 dev->data->nb_tx_queues = 0; 1256 } 1257 1258 /* 1259 * Reset and stop device. 1260 */ 1261 static int 1262 vmxnet3_dev_close(struct rte_eth_dev *dev) 1263 { 1264 int ret; 1265 PMD_INIT_FUNC_TRACE(); 1266 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1267 return 0; 1268 1269 ret = vmxnet3_dev_stop(dev); 1270 vmxnet3_free_queues(dev); 1271 1272 return ret; 1273 } 1274 1275 static int 1276 vmxnet3_dev_reset(struct rte_eth_dev *dev) 1277 { 1278 int ret; 1279 1280 ret = eth_vmxnet3_dev_uninit(dev); 1281 if (ret) 1282 return ret; 1283 ret = eth_vmxnet3_dev_init(dev); 1284 return ret; 1285 } 1286 1287 static void 1288 vmxnet3_hw_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1289 struct UPT1_TxStats *res) 1290 { 1291 #define VMXNET3_UPDATE_TX_STAT(h, i, f, r) \ 1292 ((r)->f = (h)->tqd_start[(i)].stats.f + \ 1293 (h)->saved_tx_stats[(i)].f) 1294 1295 VMXNET3_UPDATE_TX_STAT(hw, q, ucastPktsTxOK, res); 1296 VMXNET3_UPDATE_TX_STAT(hw, q, mcastPktsTxOK, res); 1297 VMXNET3_UPDATE_TX_STAT(hw, q, bcastPktsTxOK, res); 1298 VMXNET3_UPDATE_TX_STAT(hw, q, ucastBytesTxOK, res); 1299 VMXNET3_UPDATE_TX_STAT(hw, q, mcastBytesTxOK, res); 1300 VMXNET3_UPDATE_TX_STAT(hw, q, bcastBytesTxOK, res); 1301 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxError, res); 1302 VMXNET3_UPDATE_TX_STAT(hw, q, pktsTxDiscard, res); 1303 1304 #undef VMXNET3_UPDATE_TX_STAT 1305 } 1306 1307 static void 1308 vmxnet3_hw_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1309 struct UPT1_RxStats *res) 1310 { 1311 #define VMXNET3_UPDATE_RX_STAT(h, i, f, r) \ 1312 ((r)->f = (h)->rqd_start[(i)].stats.f + \ 1313 (h)->saved_rx_stats[(i)].f) 1314 1315 VMXNET3_UPDATE_RX_STAT(hw, q, ucastPktsRxOK, res); 1316 VMXNET3_UPDATE_RX_STAT(hw, q, mcastPktsRxOK, res); 1317 VMXNET3_UPDATE_RX_STAT(hw, q, bcastPktsRxOK, res); 1318 VMXNET3_UPDATE_RX_STAT(hw, q, ucastBytesRxOK, res); 1319 VMXNET3_UPDATE_RX_STAT(hw, q, mcastBytesRxOK, res); 1320 VMXNET3_UPDATE_RX_STAT(hw, q, bcastBytesRxOK, res); 1321 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxError, res); 1322 VMXNET3_UPDATE_RX_STAT(hw, q, pktsRxOutOfBuf, res); 1323 1324 #undef VMXNET3_UPDATE_RX_STAT 1325 } 1326 1327 static void 1328 vmxnet3_tx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1329 struct UPT1_TxStats *res) 1330 { 1331 vmxnet3_hw_tx_stats_get(hw, q, res); 1332 1333 #define VMXNET3_REDUCE_SNAPSHOT_TX_STAT(h, i, f, r) \ 1334 ((r)->f -= (h)->snapshot_tx_stats[(i)].f) 1335 1336 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastPktsTxOK, res); 1337 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastPktsTxOK, res); 1338 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastPktsTxOK, res); 1339 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, ucastBytesTxOK, res); 1340 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, mcastBytesTxOK, res); 1341 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, bcastBytesTxOK, res); 1342 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxError, res); 1343 VMXNET3_REDUCE_SNAPSHOT_TX_STAT(hw, q, pktsTxDiscard, res); 1344 1345 #undef VMXNET3_REDUCE_SNAPSHOT_TX_STAT 1346 } 1347 1348 static void 1349 vmxnet3_rx_stats_get(struct vmxnet3_hw *hw, unsigned int q, 1350 struct UPT1_RxStats *res) 1351 { 1352 vmxnet3_hw_rx_stats_get(hw, q, res); 1353 1354 #define VMXNET3_REDUCE_SNAPSHOT_RX_STAT(h, i, f, r) \ 1355 ((r)->f -= (h)->snapshot_rx_stats[(i)].f) 1356 1357 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastPktsRxOK, res); 1358 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastPktsRxOK, res); 1359 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastPktsRxOK, res); 1360 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, ucastBytesRxOK, res); 1361 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, mcastBytesRxOK, res); 1362 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, bcastBytesRxOK, res); 1363 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxError, res); 1364 VMXNET3_REDUCE_SNAPSHOT_RX_STAT(hw, q, pktsRxOutOfBuf, res); 1365 1366 #undef VMXNET3_REDUCE_SNAPSHOT_RX_STAT 1367 } 1368 1369 static void 1370 vmxnet3_hw_stats_save(struct vmxnet3_hw *hw) 1371 { 1372 unsigned int i; 1373 1374 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1375 1376 for (i = 0; i < hw->num_tx_queues; i++) 1377 vmxnet3_hw_tx_stats_get(hw, i, &hw->saved_tx_stats[i]); 1378 for (i = 0; i < hw->num_rx_queues; i++) 1379 vmxnet3_hw_rx_stats_get(hw, i, &hw->saved_rx_stats[i]); 1380 } 1381 1382 static int 1383 vmxnet3_dev_xstats_get_names(struct rte_eth_dev *dev, 1384 struct rte_eth_xstat_name *xstats_names, 1385 unsigned int n) 1386 { 1387 unsigned int i, t, count = 0; 1388 unsigned int nstats = 1389 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 1390 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 1391 1392 if (!xstats_names || n < nstats) 1393 return nstats; 1394 1395 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1396 if (!dev->data->rx_queues[i]) 1397 continue; 1398 1399 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 1400 snprintf(xstats_names[count].name, 1401 sizeof(xstats_names[count].name), 1402 "rx_q%u_%s", i, 1403 vmxnet3_rxq_stat_strings[t].name); 1404 count++; 1405 } 1406 } 1407 1408 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1409 if (!dev->data->tx_queues[i]) 1410 continue; 1411 1412 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 1413 snprintf(xstats_names[count].name, 1414 sizeof(xstats_names[count].name), 1415 "tx_q%u_%s", i, 1416 vmxnet3_txq_stat_strings[t].name); 1417 count++; 1418 } 1419 } 1420 1421 return count; 1422 } 1423 1424 static int 1425 vmxnet3_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1426 unsigned int n) 1427 { 1428 unsigned int i, t, count = 0; 1429 unsigned int nstats = 1430 dev->data->nb_tx_queues * RTE_DIM(vmxnet3_txq_stat_strings) + 1431 dev->data->nb_rx_queues * RTE_DIM(vmxnet3_rxq_stat_strings); 1432 1433 if (n < nstats) 1434 return nstats; 1435 1436 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1437 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i]; 1438 1439 if (rxq == NULL) 1440 continue; 1441 1442 for (t = 0; t < RTE_DIM(vmxnet3_rxq_stat_strings); t++) { 1443 xstats[count].value = *(uint64_t *)(((char *)&rxq->stats) + 1444 vmxnet3_rxq_stat_strings[t].offset); 1445 xstats[count].id = count; 1446 count++; 1447 } 1448 } 1449 1450 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1451 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i]; 1452 1453 if (txq == NULL) 1454 continue; 1455 1456 for (t = 0; t < RTE_DIM(vmxnet3_txq_stat_strings); t++) { 1457 xstats[count].value = *(uint64_t *)(((char *)&txq->stats) + 1458 vmxnet3_txq_stat_strings[t].offset); 1459 xstats[count].id = count; 1460 count++; 1461 } 1462 } 1463 1464 return count; 1465 } 1466 1467 static int 1468 vmxnet3_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1469 { 1470 unsigned int i; 1471 struct vmxnet3_hw *hw = dev->data->dev_private; 1472 struct UPT1_TxStats txStats; 1473 struct UPT1_RxStats rxStats; 1474 uint64_t packets, bytes; 1475 1476 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1477 1478 for (i = 0; i < hw->num_tx_queues; i++) { 1479 vmxnet3_tx_stats_get(hw, i, &txStats); 1480 1481 packets = txStats.ucastPktsTxOK + 1482 txStats.mcastPktsTxOK + 1483 txStats.bcastPktsTxOK; 1484 1485 bytes = txStats.ucastBytesTxOK + 1486 txStats.mcastBytesTxOK + 1487 txStats.bcastBytesTxOK; 1488 1489 stats->opackets += packets; 1490 stats->obytes += bytes; 1491 stats->oerrors += txStats.pktsTxError + txStats.pktsTxDiscard; 1492 1493 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 1494 stats->q_opackets[i] = packets; 1495 stats->q_obytes[i] = bytes; 1496 } 1497 } 1498 1499 for (i = 0; i < hw->num_rx_queues; i++) { 1500 vmxnet3_rx_stats_get(hw, i, &rxStats); 1501 1502 packets = rxStats.ucastPktsRxOK + 1503 rxStats.mcastPktsRxOK + 1504 rxStats.bcastPktsRxOK; 1505 1506 bytes = rxStats.ucastBytesRxOK + 1507 rxStats.mcastBytesRxOK + 1508 rxStats.bcastBytesRxOK; 1509 1510 stats->ipackets += packets; 1511 stats->ibytes += bytes; 1512 stats->ierrors += rxStats.pktsRxError; 1513 stats->imissed += rxStats.pktsRxOutOfBuf; 1514 1515 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 1516 stats->q_ipackets[i] = packets; 1517 stats->q_ibytes[i] = bytes; 1518 stats->q_errors[i] = rxStats.pktsRxError; 1519 } 1520 } 1521 1522 return 0; 1523 } 1524 1525 static int 1526 vmxnet3_dev_stats_reset(struct rte_eth_dev *dev) 1527 { 1528 unsigned int i; 1529 struct vmxnet3_hw *hw = dev->data->dev_private; 1530 struct UPT1_TxStats txStats = {0}; 1531 struct UPT1_RxStats rxStats = {0}; 1532 1533 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS); 1534 1535 for (i = 0; i < hw->num_tx_queues; i++) { 1536 vmxnet3_hw_tx_stats_get(hw, i, &txStats); 1537 memcpy(&hw->snapshot_tx_stats[i], &txStats, 1538 sizeof(hw->snapshot_tx_stats[0])); 1539 } 1540 for (i = 0; i < hw->num_rx_queues; i++) { 1541 vmxnet3_hw_rx_stats_get(hw, i, &rxStats); 1542 memcpy(&hw->snapshot_rx_stats[i], &rxStats, 1543 sizeof(hw->snapshot_rx_stats[0])); 1544 } 1545 1546 return 0; 1547 } 1548 1549 static int 1550 vmxnet3_dev_info_get(struct rte_eth_dev *dev, 1551 struct rte_eth_dev_info *dev_info) 1552 { 1553 struct vmxnet3_hw *hw = dev->data->dev_private; 1554 int queues = 0; 1555 1556 if (VMXNET3_VERSION_GE_6(hw)) { 1557 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1558 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 1559 queues = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1560 1561 if (queues > 0) { 1562 dev_info->max_rx_queues = 1563 RTE_MIN(VMXNET3_EXT_MAX_RX_QUEUES, ((queues >> 8) & 0xff)); 1564 dev_info->max_tx_queues = 1565 RTE_MIN(VMXNET3_EXT_MAX_TX_QUEUES, (queues & 0xff)); 1566 } else { 1567 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; 1568 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; 1569 } 1570 } else { 1571 dev_info->max_rx_queues = VMXNET3_MAX_RX_QUEUES; 1572 dev_info->max_tx_queues = VMXNET3_MAX_TX_QUEUES; 1573 } 1574 1575 dev_info->min_rx_bufsize = 1518 + RTE_PKTMBUF_HEADROOM; 1576 dev_info->max_rx_pktlen = 16384; /* includes CRC, cf MAXFRS register */ 1577 dev_info->min_mtu = VMXNET3_MIN_MTU; 1578 dev_info->max_mtu = VMXNET3_VERSION_GE_6(hw) ? VMXNET3_V6_MAX_MTU : VMXNET3_MAX_MTU; 1579 dev_info->speed_capa = RTE_ETH_LINK_SPEED_10G; 1580 dev_info->max_mac_addrs = VMXNET3_MAX_MAC_ADDRS; 1581 1582 dev_info->flow_type_rss_offloads = VMXNET3_RSS_OFFLOAD_ALL; 1583 1584 if (VMXNET3_VERSION_GE_4(hw)) { 1585 dev_info->flow_type_rss_offloads |= VMXNET3_V4_RSS_MASK; 1586 } 1587 1588 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 1589 .nb_max = VMXNET3_RX_RING_MAX_SIZE, 1590 .nb_min = VMXNET3_DEF_RX_RING_SIZE, 1591 .nb_align = 1, 1592 }; 1593 1594 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 1595 .nb_max = VMXNET3_TX_RING_MAX_SIZE, 1596 .nb_min = VMXNET3_DEF_TX_RING_SIZE, 1597 .nb_align = 1, 1598 .nb_seg_max = VMXNET3_TX_MAX_SEG, 1599 .nb_mtu_seg_max = VMXNET3_MAX_TXD_PER_PKT, 1600 }; 1601 1602 dev_info->rx_offload_capa = VMXNET3_RX_OFFLOAD_CAP; 1603 dev_info->rx_queue_offload_capa = 0; 1604 dev_info->tx_offload_capa = VMXNET3_TX_OFFLOAD_CAP; 1605 dev_info->tx_queue_offload_capa = 0; 1606 if (hw->rss_conf == NULL) { 1607 /* RSS not configured */ 1608 dev_info->reta_size = 0; 1609 } else { 1610 dev_info->reta_size = hw->rss_conf->indTableSize; 1611 } 1612 return 0; 1613 } 1614 1615 static int 1616 vmxnet3_hw_ver_get(struct rte_eth_dev *dev, 1617 char *fw_version, size_t fw_size) 1618 { 1619 int ret; 1620 struct vmxnet3_hw *hw = dev->data->dev_private; 1621 1622 ret = snprintf(fw_version, fw_size, "v%d", hw->version); 1623 1624 ret += 1; /* add the size of '\0' */ 1625 if (fw_size < (uint32_t)ret) 1626 return ret; 1627 else 1628 return 0; 1629 } 1630 1631 static const uint32_t * 1632 vmxnet3_dev_supported_ptypes_get(struct rte_eth_dev *dev, 1633 size_t *no_of_elements) 1634 { 1635 static const uint32_t ptypes[] = { 1636 RTE_PTYPE_L3_IPV4_EXT, 1637 RTE_PTYPE_L3_IPV4, 1638 }; 1639 1640 if (dev->rx_pkt_burst == vmxnet3_recv_pkts) { 1641 *no_of_elements = RTE_DIM(ptypes); 1642 return ptypes; 1643 } 1644 return NULL; 1645 } 1646 1647 static int 1648 vmxnet3_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) 1649 { 1650 struct vmxnet3_hw *hw = dev->data->dev_private; 1651 1652 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)(hw->perm_addr)); 1653 vmxnet3_write_mac(hw, mac_addr->addr_bytes); 1654 return 0; 1655 } 1656 1657 static int 1658 vmxnet3_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1659 { 1660 struct vmxnet3_hw *hw = dev->data->dev_private; 1661 uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 4; 1662 1663 if (mtu < VMXNET3_MIN_MTU) 1664 return -EINVAL; 1665 1666 if (VMXNET3_VERSION_GE_6(hw)) { 1667 if (mtu > VMXNET3_V6_MAX_MTU) 1668 return -EINVAL; 1669 } else { 1670 if (mtu > VMXNET3_MAX_MTU) { 1671 PMD_DRV_LOG(ERR, "MTU %d too large in device version v%d", 1672 mtu, hw->version); 1673 return -EINVAL; 1674 } 1675 } 1676 1677 dev->data->mtu = mtu; 1678 /* update max frame size */ 1679 dev->data->dev_conf.rxmode.mtu = frame_size; 1680 1681 if (dev->data->dev_started == 0) 1682 return 0; 1683 1684 /* changing mtu for vmxnet3 pmd does not require a restart 1685 * as it does not need to repopulate the rx rings to support 1686 * different mtu size. We stop and restart the device here 1687 * just to pass the mtu info to the backend. 1688 */ 1689 vmxnet3_dev_stop(dev); 1690 vmxnet3_dev_start(dev); 1691 1692 return 0; 1693 } 1694 1695 /* return 0 means link status changed, -1 means not changed */ 1696 static int 1697 __vmxnet3_dev_link_update(struct rte_eth_dev *dev, 1698 __rte_unused int wait_to_complete) 1699 { 1700 struct vmxnet3_hw *hw = dev->data->dev_private; 1701 struct rte_eth_link link; 1702 uint32_t ret; 1703 1704 memset(&link, 0, sizeof(link)); 1705 1706 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 1707 ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); 1708 1709 if (ret & 0x1) 1710 link.link_status = RTE_ETH_LINK_UP; 1711 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; 1712 link.link_speed = RTE_ETH_SPEED_NUM_10G; 1713 link.link_autoneg = RTE_ETH_LINK_FIXED; 1714 1715 return rte_eth_linkstatus_set(dev, &link); 1716 } 1717 1718 static int 1719 vmxnet3_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) 1720 { 1721 /* Link status doesn't change for stopped dev */ 1722 if (dev->data->dev_started == 0) 1723 return -1; 1724 1725 return __vmxnet3_dev_link_update(dev, wait_to_complete); 1726 } 1727 1728 /* Updating rxmode through Vmxnet3_DriverShared structure in adapter */ 1729 static void 1730 vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set) 1731 { 1732 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1733 1734 if (set) 1735 rxConf->rxMode = rxConf->rxMode | feature; 1736 else 1737 rxConf->rxMode = rxConf->rxMode & (~feature); 1738 1739 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE); 1740 } 1741 1742 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1743 static int 1744 vmxnet3_dev_promiscuous_enable(struct rte_eth_dev *dev) 1745 { 1746 struct vmxnet3_hw *hw = dev->data->dev_private; 1747 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1748 1749 memset(vf_table, 0, VMXNET3_VFT_TABLE_SIZE); 1750 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 1); 1751 1752 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1753 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1754 1755 return 0; 1756 } 1757 1758 /* Promiscuous supported only if Vmxnet3_DriverShared is initialized in adapter */ 1759 static int 1760 vmxnet3_dev_promiscuous_disable(struct rte_eth_dev *dev) 1761 { 1762 struct vmxnet3_hw *hw = dev->data->dev_private; 1763 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; 1764 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 1765 1766 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) 1767 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1768 else 1769 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1770 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_PROMISC, 0); 1771 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1772 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1773 1774 return 0; 1775 } 1776 1777 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1778 static int 1779 vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev) 1780 { 1781 struct vmxnet3_hw *hw = dev->data->dev_private; 1782 1783 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1); 1784 1785 return 0; 1786 } 1787 1788 /* Allmulticast supported only if Vmxnet3_DriverShared is initialized in adapter */ 1789 static int 1790 vmxnet3_dev_allmulticast_disable(struct rte_eth_dev *dev) 1791 { 1792 struct vmxnet3_hw *hw = dev->data->dev_private; 1793 1794 vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 0); 1795 1796 return 0; 1797 } 1798 1799 /* Enable/disable filter on vlan */ 1800 static int 1801 vmxnet3_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vid, int on) 1802 { 1803 struct vmxnet3_hw *hw = dev->data->dev_private; 1804 struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf; 1805 uint32_t *vf_table = rxConf->vfTable; 1806 1807 /* save state for restore */ 1808 if (on) 1809 VMXNET3_SET_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1810 else 1811 VMXNET3_CLEAR_VFTABLE_ENTRY(hw->shadow_vfta, vid); 1812 1813 /* don't change active filter if in promiscuous mode */ 1814 if (rxConf->rxMode & VMXNET3_RXM_PROMISC) 1815 return 0; 1816 1817 /* set in hardware */ 1818 if (on) 1819 VMXNET3_SET_VFTABLE_ENTRY(vf_table, vid); 1820 else 1821 VMXNET3_CLEAR_VFTABLE_ENTRY(vf_table, vid); 1822 1823 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1824 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1825 return 0; 1826 } 1827 1828 static int 1829 vmxnet3_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1830 { 1831 struct vmxnet3_hw *hw = dev->data->dev_private; 1832 Vmxnet3_DSDevRead *devRead = &hw->shared->devRead; 1833 uint32_t *vf_table = devRead->rxFilterConf.vfTable; 1834 uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; 1835 1836 if (mask & RTE_ETH_VLAN_STRIP_MASK) { 1837 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 1838 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 1839 else 1840 devRead->misc.uptFeatures &= ~UPT1_F_RXVLAN; 1841 1842 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1843 VMXNET3_CMD_UPDATE_FEATURE); 1844 } 1845 1846 if (mask & RTE_ETH_VLAN_FILTER_MASK) { 1847 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) 1848 memcpy(vf_table, hw->shadow_vfta, VMXNET3_VFT_TABLE_SIZE); 1849 else 1850 memset(vf_table, 0xff, VMXNET3_VFT_TABLE_SIZE); 1851 1852 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1853 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1854 } 1855 1856 return 0; 1857 } 1858 1859 static void 1860 vmxnet3_process_events(struct rte_eth_dev *dev) 1861 { 1862 struct vmxnet3_hw *hw = dev->data->dev_private; 1863 uint32_t events = hw->shared->ecr; 1864 int i; 1865 1866 if (!events) 1867 return; 1868 1869 /* 1870 * ECR bits when written with 1b are cleared. Hence write 1871 * events back to ECR so that the bits which were set will be reset. 1872 */ 1873 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_ECR, events); 1874 1875 /* Check if link state has changed */ 1876 if (events & VMXNET3_ECR_LINK) { 1877 PMD_DRV_LOG(DEBUG, "Process events: VMXNET3_ECR_LINK event"); 1878 if (vmxnet3_dev_link_update(dev, 0) == 0) 1879 rte_eth_dev_callback_process(dev, 1880 RTE_ETH_EVENT_INTR_LSC, 1881 NULL); 1882 } 1883 1884 /* Check if there is an error on xmit/recv queues */ 1885 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 1886 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 1887 VMXNET3_CMD_GET_QUEUE_STATUS); 1888 1889 PMD_DRV_LOG(ERR, "queue error event 0x%x for " 1890 RTE_ETHER_ADDR_PRT_FMT, events, 1891 hw->perm_addr[0], hw->perm_addr[1], 1892 hw->perm_addr[2], hw->perm_addr[3], 1893 hw->perm_addr[4], hw->perm_addr[5]); 1894 1895 for (i = 0; i < hw->num_tx_queues; i++) { 1896 if (hw->tqd_start[i].status.stopped) 1897 PMD_DRV_LOG(ERR, "tq %d error 0x%x", 1898 i, hw->tqd_start[i].status.error); 1899 } 1900 for (i = 0; i < hw->num_rx_queues; i++) { 1901 if (hw->rqd_start[i].status.stopped) 1902 PMD_DRV_LOG(ERR, "rq %d error 0x%x", 1903 i, hw->rqd_start[i].status.error); 1904 } 1905 1906 /* Have to reset the device */ 1907 /* Notify the application so that it can reset the device */ 1908 rte_eth_dev_callback_process(dev, 1909 RTE_ETH_EVENT_INTR_RESET, 1910 NULL); 1911 } 1912 1913 if (events & VMXNET3_ECR_DIC) 1914 PMD_DRV_LOG(DEBUG, "Device implementation change event."); 1915 1916 if (events & VMXNET3_ECR_DEBUG) 1917 PMD_DRV_LOG(DEBUG, "Debug event generated by device."); 1918 } 1919 1920 static void 1921 vmxnet3_interrupt_handler(void *param) 1922 { 1923 struct rte_eth_dev *dev = param; 1924 struct vmxnet3_hw *hw = dev->data->dev_private; 1925 uint32_t events; 1926 uint8 *eventIntrIdx; 1927 uint32 *intrCtrl; 1928 1929 PMD_INIT_FUNC_TRACE(); 1930 1931 vmxnet3_get_intr_ctrl_ev(hw, &eventIntrIdx, &intrCtrl); 1932 vmxnet3_disable_intr(hw, *eventIntrIdx); 1933 1934 events = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_ECR); 1935 if (events == 0) 1936 goto done; 1937 1938 PMD_DRV_LOG(DEBUG, "Reading events: 0x%X", events); 1939 vmxnet3_process_events(dev); 1940 done: 1941 vmxnet3_enable_intr(hw, *eventIntrIdx); 1942 } 1943 1944 static int 1945 vmxnet3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1946 { 1947 #ifndef RTE_EXEC_ENV_FREEBSD 1948 struct vmxnet3_hw *hw = dev->data->dev_private; 1949 1950 vmxnet3_enable_intr(hw, 1951 rte_intr_vec_list_index_get(dev->intr_handle, 1952 queue_id)); 1953 #endif 1954 1955 return 0; 1956 } 1957 1958 static int 1959 vmxnet3_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1960 { 1961 struct vmxnet3_hw *hw = dev->data->dev_private; 1962 1963 vmxnet3_disable_intr(hw, 1964 rte_intr_vec_list_index_get(dev->intr_handle, queue_id)); 1965 1966 return 0; 1967 } 1968 1969 RTE_PMD_REGISTER_PCI(net_vmxnet3, rte_vmxnet3_pmd); 1970 RTE_PMD_REGISTER_PCI_TABLE(net_vmxnet3, pci_id_vmxnet3_map); 1971 RTE_PMD_REGISTER_KMOD_DEP(net_vmxnet3, "* igb_uio | uio_pci_generic | vfio-pci"); 1972 RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_init, init, NOTICE); 1973 RTE_LOG_REGISTER_SUFFIX(vmxnet3_logtype_driver, driver, NOTICE); 1974 1975 static int 1976 vmxnet3_rss_reta_update(struct rte_eth_dev *dev, 1977 struct rte_eth_rss_reta_entry64 *reta_conf, 1978 uint16_t reta_size) 1979 { 1980 int i, idx, shift; 1981 struct vmxnet3_hw *hw = dev->data->dev_private; 1982 struct VMXNET3_RSSConf *dev_rss_conf = hw->rss_conf; 1983 1984 if (reta_size != dev_rss_conf->indTableSize) { 1985 PMD_DRV_LOG(ERR, 1986 "The size of hash lookup table configured (%d) doesn't match " 1987 "the supported number (%d)", 1988 reta_size, dev_rss_conf->indTableSize); 1989 return -EINVAL; 1990 } 1991 1992 for (i = 0; i < reta_size; i++) { 1993 idx = i / RTE_ETH_RETA_GROUP_SIZE; 1994 shift = i % RTE_ETH_RETA_GROUP_SIZE; 1995 if (reta_conf[idx].mask & RTE_BIT64(shift)) 1996 dev_rss_conf->indTable[i] = (uint8_t)reta_conf[idx].reta[shift]; 1997 } 1998 1999 VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, 2000 VMXNET3_CMD_UPDATE_RSSIDT); 2001 2002 return 0; 2003 } 2004 2005 static int 2006 vmxnet3_rss_reta_query(struct rte_eth_dev *dev, 2007 struct rte_eth_rss_reta_entry64 *reta_conf, 2008 uint16_t reta_size) 2009 { 2010 int i, idx, shift; 2011 struct vmxnet3_hw *hw = dev->data->dev_private; 2012 struct VMXNET3_RSSConf *dev_rss_conf = hw->rss_conf; 2013 2014 if (reta_size != dev_rss_conf->indTableSize) { 2015 PMD_DRV_LOG(ERR, 2016 "Size of requested hash lookup table (%d) doesn't " 2017 "match the configured size (%d)", 2018 reta_size, dev_rss_conf->indTableSize); 2019 return -EINVAL; 2020 } 2021 2022 for (i = 0; i < reta_size; i++) { 2023 idx = i / RTE_ETH_RETA_GROUP_SIZE; 2024 shift = i % RTE_ETH_RETA_GROUP_SIZE; 2025 if (reta_conf[idx].mask & RTE_BIT64(shift)) 2026 reta_conf[idx].reta[shift] = dev_rss_conf->indTable[i]; 2027 } 2028 2029 return 0; 2030 } 2031