1 /* $OpenBSD: if_vmx.c,v 1.90 2025/01/24 10:29:43 yasuoka Exp $ */ 2 3 /* 4 * Copyright (c) 2013 Tsubai Masanari 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include "bpfilter.h" 20 #include "kstat.h" 21 #include "vlan.h" 22 23 #include <sys/param.h> 24 #include <sys/device.h> 25 #include <sys/mbuf.h> 26 #include <sys/socket.h> 27 #include <sys/sockio.h> 28 #include <sys/systm.h> 29 #include <sys/atomic.h> 30 #include <sys/intrmap.h> 31 #include <sys/kstat.h> 32 33 #include <net/bpf.h> 34 #include <net/if.h> 35 #include <net/toeplitz.h> 36 #include <net/if_media.h> 37 #include <net/route.h> 38 39 #include <netinet/in.h> 40 #include <netinet/if_ether.h> 41 #include <netinet/ip.h> 42 #include <netinet/tcp.h> 43 #include <netinet/tcp_timer.h> 44 #include <netinet/tcp_var.h> 45 #include <netinet/udp.h> 46 47 #include <machine/bus.h> 48 49 #include <dev/pci/if_vmxreg.h> 50 #include <dev/pci/pcivar.h> 51 #include <dev/pci/pcidevs.h> 52 53 #define VMX_MAX_QUEUES MIN(VMXNET3_MAX_TX_QUEUES, VMXNET3_MAX_RX_QUEUES) 54 55 #define NTXDESC 512 /* tx ring size */ 56 #define NTXSEGS 8 /* tx descriptors per packet */ 57 #define NRXDESC 512 58 #define NTXCOMPDESC NTXDESC 59 #define NRXCOMPDESC (NRXDESC * 2) /* ring1 + ring2 */ 60 61 #define VMXNET3_DRIVER_VERSION 0x00010000 62 63 #define VMX_TX_GEN htole32(VMXNET3_TX_GEN_M << VMXNET3_TX_GEN_S) 64 #define VMX_TXC_GEN htole32(VMXNET3_TXC_GEN_M << VMXNET3_TXC_GEN_S) 65 #define VMX_RX_GEN htole32(VMXNET3_RX_GEN_M << VMXNET3_RX_GEN_S) 66 #define VMX_RXC_GEN htole32(VMXNET3_RXC_GEN_M << VMXNET3_RXC_GEN_S) 67 68 struct vmx_dmamem { 69 bus_dmamap_t vdm_map; 70 bus_dma_segment_t vdm_seg; 71 int vdm_nsegs; 72 size_t vdm_size; 73 caddr_t vdm_kva; 74 }; 75 76 #define VMX_DMA_MAP(_vdm) ((_vdm)->vdm_map) 77 #define VMX_DMA_DVA(_vdm) ((_vdm)->vdm_map->dm_segs[0].ds_addr) 78 #define VMX_DMA_KVA(_vdm) ((void *)(_vdm)->vdm_kva) 79 #define VMX_DMA_LEN(_vdm) ((_vdm)->vdm_size) 80 81 struct vmxnet3_softc; 82 83 struct vmxnet3_txring { 84 struct vmx_dmamem dmamem; 85 struct mbuf *m[NTXDESC]; 86 bus_dmamap_t dmap[NTXDESC]; 87 struct vmxnet3_txdesc *txd; 88 u_int32_t gen; 89 volatile u_int prod; 90 volatile u_int cons; 91 }; 92 93 struct vmxnet3_rxring { 94 struct vmxnet3_softc *sc; 95 struct vmxnet3_rxq_shared *rs; /* copy of the rxqueue rs */ 96 struct vmx_dmamem dmamem; 97 struct mbuf *m[NRXDESC]; 98 bus_dmamap_t dmap[NRXDESC]; 99 struct mutex mtx; 100 struct if_rxring rxr; 101 struct timeout refill; 102 struct vmxnet3_rxdesc *rxd; 103 bus_size_t rxh; 104 u_int fill; 105 u_int32_t gen; 106 u_int8_t rid; 107 }; 108 109 struct vmxnet3_comp_ring { 110 struct vmx_dmamem dmamem; 111 union { 112 struct vmxnet3_txcompdesc *txcd; 113 struct vmxnet3_rxcompdesc *rxcd; 114 }; 115 u_int next; 116 u_int32_t gen; 117 struct mbuf *sendmp; 118 struct mbuf *lastmp; 119 }; 120 121 struct vmxnet3_txqueue { 122 struct vmxnet3_softc *sc; /* sigh */ 123 struct vmxnet3_txring cmd_ring; 124 struct vmxnet3_comp_ring comp_ring; 125 struct vmxnet3_txq_shared *ts; 126 struct ifqueue *ifq; 127 struct kstat *txkstat; 128 unsigned int queue; 129 } __aligned(64); 130 131 struct vmxnet3_rxqueue { 132 struct vmxnet3_softc *sc; /* sigh */ 133 struct vmxnet3_rxring cmd_ring[2]; 134 struct vmxnet3_comp_ring comp_ring; 135 struct vmxnet3_rxq_shared *rs; 136 struct ifiqueue *ifiq; 137 struct kstat *rxkstat; 138 } __aligned(64); 139 140 struct vmxnet3_queue { 141 struct vmxnet3_txqueue tx; 142 struct vmxnet3_rxqueue rx; 143 struct vmxnet3_softc *sc; 144 char intrname[16]; 145 void *ih; 146 int intr; 147 }; 148 149 struct vmxnet3_softc { 150 struct device sc_dev; 151 struct arpcom sc_arpcom; 152 struct ifmedia sc_media; 153 154 bus_space_tag_t sc_iot0; 155 bus_space_tag_t sc_iot1; 156 bus_space_handle_t sc_ioh0; 157 bus_space_handle_t sc_ioh1; 158 bus_dma_tag_t sc_dmat; 159 void *sc_ih; 160 161 int sc_nqueues; 162 struct vmxnet3_queue *sc_q; 163 struct intrmap *sc_intrmap; 164 165 u_int sc_vrrs; 166 struct vmxnet3_driver_shared *sc_ds; 167 u_int8_t *sc_mcast; 168 struct vmxnet3_upt1_rss_conf *sc_rss; 169 170 #if NKSTAT > 0 171 struct rwlock sc_kstat_lock; 172 struct timeval sc_kstat_updated; 173 #endif 174 }; 175 176 #define JUMBO_LEN ((16 * 1024) - 1) 177 #define DMAADDR(map) ((map)->dm_segs[0].ds_addr) 178 179 #define READ_BAR0(sc, reg) bus_space_read_4((sc)->sc_iot0, (sc)->sc_ioh0, reg) 180 #define READ_BAR1(sc, reg) bus_space_read_4((sc)->sc_iot1, (sc)->sc_ioh1, reg) 181 #define WRITE_BAR0(sc, reg, val) \ 182 bus_space_write_4((sc)->sc_iot0, (sc)->sc_ioh0, reg, val) 183 #define WRITE_BAR1(sc, reg, val) \ 184 bus_space_write_4((sc)->sc_iot1, (sc)->sc_ioh1, reg, val) 185 #define WRITE_CMD(sc, cmd) WRITE_BAR1(sc, VMXNET3_BAR1_CMD, cmd) 186 187 int vmxnet3_match(struct device *, void *, void *); 188 void vmxnet3_attach(struct device *, struct device *, void *); 189 int vmxnet3_dma_init(struct vmxnet3_softc *); 190 int vmxnet3_alloc_txring(struct vmxnet3_softc *, int, int); 191 int vmxnet3_alloc_rxring(struct vmxnet3_softc *, int, int); 192 void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 193 void vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 194 void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 195 void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 196 void vmxnet3_link_state(struct vmxnet3_softc *); 197 void vmxnet3_enable_all_intrs(struct vmxnet3_softc *); 198 void vmxnet3_disable_all_intrs(struct vmxnet3_softc *); 199 int vmxnet3_intr(void *); 200 int vmxnet3_intr_intx(void *); 201 int vmxnet3_intr_event(void *); 202 int vmxnet3_intr_queue(void *); 203 void vmxnet3_evintr(struct vmxnet3_softc *); 204 void vmxnet3_txintr(struct vmxnet3_softc *, struct vmxnet3_txqueue *); 205 void vmxnet3_rxintr(struct vmxnet3_softc *, struct vmxnet3_rxqueue *); 206 void vmxnet3_rxfill_tick(void *); 207 void vmxnet3_rxfill(struct vmxnet3_rxring *); 208 void vmxnet3_iff(struct vmxnet3_softc *); 209 void vmxnet3_rx_offload(struct vmxnet3_rxcompdesc *, struct mbuf *); 210 void vmxnet3_stop(struct ifnet *); 211 void vmxnet3_reset(struct vmxnet3_softc *); 212 int vmxnet3_init(struct vmxnet3_softc *); 213 int vmxnet3_ioctl(struct ifnet *, u_long, caddr_t); 214 void vmxnet3_start(struct ifqueue *); 215 void vmxnet3_watchdog(struct ifnet *); 216 void vmxnet3_media_status(struct ifnet *, struct ifmediareq *); 217 int vmxnet3_media_change(struct ifnet *); 218 void *vmxnet3_dma_allocmem(struct vmxnet3_softc *, u_int, u_int, bus_addr_t *); 219 220 static int vmx_dmamem_alloc(struct vmxnet3_softc *, struct vmx_dmamem *, 221 bus_size_t, u_int); 222 #ifdef notyet 223 static void vmx_dmamem_free(struct vmxnet3_softc *, struct vmx_dmamem *); 224 #endif 225 226 #if NKSTAT > 0 227 static void vmx_kstat_init(struct vmxnet3_softc *); 228 static void vmx_kstat_txstats(struct vmxnet3_softc *, 229 struct vmxnet3_txqueue *, int); 230 static void vmx_kstat_rxstats(struct vmxnet3_softc *, 231 struct vmxnet3_rxqueue *, int); 232 #endif /* NKSTAT > 0 */ 233 234 const struct pci_matchid vmx_devices[] = { 235 { PCI_VENDOR_VMWARE, PCI_PRODUCT_VMWARE_NET_3 } 236 }; 237 238 const struct cfattach vmx_ca = { 239 sizeof(struct vmxnet3_softc), vmxnet3_match, vmxnet3_attach 240 }; 241 242 struct cfdriver vmx_cd = { 243 NULL, "vmx", DV_IFNET 244 }; 245 246 int 247 vmxnet3_match(struct device *parent, void *match, void *aux) 248 { 249 return (pci_matchbyid(aux, vmx_devices, nitems(vmx_devices))); 250 } 251 252 void 253 vmxnet3_attach(struct device *parent, struct device *self, void *aux) 254 { 255 struct vmxnet3_softc *sc = (void *)self; 256 struct pci_attach_args *pa = aux; 257 struct ifnet *ifp = &sc->sc_arpcom.ac_if; 258 pci_intr_handle_t ih; 259 const char *intrstr; 260 u_int memtype, ver, macl, mach, intrcfg; 261 u_char enaddr[ETHER_ADDR_LEN]; 262 int (*isr)(void *); 263 int msix = 0; 264 int i; 265 266 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x10); 267 if (pci_mapreg_map(pa, 0x10, memtype, 0, &sc->sc_iot0, &sc->sc_ioh0, 268 NULL, NULL, 0)) { 269 printf(": failed to map BAR0\n"); 270 return; 271 } 272 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x14); 273 if (pci_mapreg_map(pa, 0x14, memtype, 0, &sc->sc_iot1, &sc->sc_ioh1, 274 NULL, NULL, 0)) { 275 printf(": failed to map BAR1\n"); 276 return; 277 } 278 279 /* Vmxnet3 Revision Report and Selection */ 280 ver = READ_BAR1(sc, VMXNET3_BAR1_VRRS); 281 if (ISSET(ver, 0x2)) { 282 sc->sc_vrrs = 2; 283 } else if (ISSET(ver, 0x1)) { 284 sc->sc_vrrs = 1; 285 } else { 286 printf(": unsupported hardware version 0x%x\n", ver); 287 return; 288 } 289 WRITE_BAR1(sc, VMXNET3_BAR1_VRRS, sc->sc_vrrs); 290 291 /* UPT Version Report and Selection */ 292 ver = READ_BAR1(sc, VMXNET3_BAR1_UVRS); 293 if (!ISSET(ver, 0x1)) { 294 printf(": incompatible UPT version 0x%x\n", ver); 295 return; 296 } 297 WRITE_BAR1(sc, VMXNET3_BAR1_UVRS, 1); 298 299 sc->sc_dmat = pa->pa_dmat; 300 301 WRITE_CMD(sc, VMXNET3_CMD_GET_INTRCFG); 302 intrcfg = READ_BAR1(sc, VMXNET3_BAR1_CMD); 303 isr = vmxnet3_intr; 304 sc->sc_nqueues = 1; 305 306 switch (intrcfg & VMXNET3_INTRCFG_TYPE_MASK) { 307 case VMXNET3_INTRCFG_TYPE_AUTO: 308 case VMXNET3_INTRCFG_TYPE_MSIX: 309 msix = pci_intr_msix_count(pa); 310 if (msix > 0) { 311 if (pci_intr_map_msix(pa, 0, &ih) == 0) { 312 msix--; /* are there spares for tx/rx qs? */ 313 if (msix == 0) 314 break; 315 316 isr = vmxnet3_intr_event; 317 sc->sc_intrmap = intrmap_create(&sc->sc_dev, 318 msix, VMX_MAX_QUEUES, INTRMAP_POWEROF2); 319 sc->sc_nqueues = intrmap_count(sc->sc_intrmap); 320 } 321 break; 322 } 323 324 /* FALLTHROUGH */ 325 case VMXNET3_INTRCFG_TYPE_MSI: 326 if (pci_intr_map_msi(pa, &ih) == 0) 327 break; 328 329 /* FALLTHROUGH */ 330 case VMXNET3_INTRCFG_TYPE_INTX: 331 isr = vmxnet3_intr_intx; 332 if (pci_intr_map(pa, &ih) == 0) 333 break; 334 335 printf(": failed to map interrupt\n"); 336 return; 337 } 338 intrstr = pci_intr_string(pa->pa_pc, ih); 339 sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 340 isr, sc, self->dv_xname); 341 if (sc->sc_ih == NULL) { 342 printf(": unable to establish interrupt handler"); 343 if (intrstr != NULL) 344 printf(" at %s", intrstr); 345 printf("\n"); 346 return; 347 } 348 if (intrstr) 349 printf(": %s", intrstr); 350 351 sc->sc_q = mallocarray(sc->sc_nqueues, sizeof(*sc->sc_q), 352 M_DEVBUF, M_WAITOK|M_ZERO); 353 354 if (sc->sc_intrmap != NULL) { 355 for (i = 0; i < sc->sc_nqueues; i++) { 356 struct vmxnet3_queue *q; 357 int vec; 358 359 q = &sc->sc_q[i]; 360 vec = i + 1; 361 if (pci_intr_map_msix(pa, vec, &ih) != 0) { 362 printf(", failed to map interrupt %d\n", vec); 363 return; 364 } 365 snprintf(q->intrname, sizeof(q->intrname), "%s:%d", 366 self->dv_xname, i); 367 q->ih = pci_intr_establish_cpu(pa->pa_pc, ih, 368 IPL_NET | IPL_MPSAFE, 369 intrmap_cpu(sc->sc_intrmap, i), 370 vmxnet3_intr_queue, q, q->intrname); 371 if (q->ih == NULL) { 372 printf(": unable to establish interrupt %d\n", 373 vec); 374 return; 375 } 376 377 q->intr = vec; 378 q->sc = sc; 379 } 380 } 381 382 if (vmxnet3_dma_init(sc)) { 383 printf(": failed to setup DMA\n"); 384 return; 385 } 386 387 printf(", %d queue%s", sc->sc_nqueues, sc->sc_nqueues > 1 ? "s" : ""); 388 389 WRITE_CMD(sc, VMXNET3_CMD_GET_MACL); 390 macl = READ_BAR1(sc, VMXNET3_BAR1_CMD); 391 enaddr[0] = macl; 392 enaddr[1] = macl >> 8; 393 enaddr[2] = macl >> 16; 394 enaddr[3] = macl >> 24; 395 WRITE_CMD(sc, VMXNET3_CMD_GET_MACH); 396 mach = READ_BAR1(sc, VMXNET3_BAR1_CMD); 397 enaddr[4] = mach; 398 enaddr[5] = mach >> 8; 399 400 WRITE_BAR1(sc, VMXNET3_BAR1_MACL, macl); 401 WRITE_BAR1(sc, VMXNET3_BAR1_MACH, mach); 402 printf(", address %s\n", ether_sprintf(enaddr)); 403 404 bcopy(enaddr, sc->sc_arpcom.ac_enaddr, 6); 405 strlcpy(ifp->if_xname, self->dv_xname, IFNAMSIZ); 406 ifp->if_softc = sc; 407 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 408 ifp->if_xflags = IFXF_MPSAFE; 409 ifp->if_ioctl = vmxnet3_ioctl; 410 ifp->if_qstart = vmxnet3_start; 411 ifp->if_watchdog = vmxnet3_watchdog; 412 ifp->if_hardmtu = VMXNET3_MAX_MTU; 413 ifp->if_capabilities = IFCAP_VLAN_MTU; 414 415 if (sc->sc_ds->upt_features & UPT1_F_CSUM) { 416 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 417 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6; 418 } 419 420 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 421 422 if (sc->sc_vrrs == 2) { 423 ifp->if_xflags |= IFXF_LRO; 424 ifp->if_capabilities |= IFCAP_LRO; 425 } 426 427 #if NVLAN > 0 428 if (sc->sc_ds->upt_features & UPT1_F_VLAN) 429 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 430 #endif 431 432 ifq_init_maxlen(&ifp->if_snd, NTXDESC); 433 434 ifmedia_init(&sc->sc_media, IFM_IMASK, vmxnet3_media_change, 435 vmxnet3_media_status); 436 ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_AUTO, 0, NULL); 437 ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_10G_T|IFM_FDX, 0, NULL); 438 ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_10G_T, 0, NULL); 439 ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_1000_T|IFM_FDX, 0, NULL); 440 ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_1000_T, 0, NULL); 441 ifmedia_set(&sc->sc_media, IFM_ETHER|IFM_AUTO); 442 443 if_attach(ifp); 444 ether_ifattach(ifp); 445 vmxnet3_link_state(sc); 446 447 if_attach_queues(ifp, sc->sc_nqueues); 448 if_attach_iqueues(ifp, sc->sc_nqueues); 449 450 #if NKSTAT > 0 451 vmx_kstat_init(sc); 452 #endif 453 454 for (i = 0; i < sc->sc_nqueues; i++) { 455 ifp->if_ifqs[i]->ifq_softc = &sc->sc_q[i].tx; 456 sc->sc_q[i].tx.ifq = ifp->if_ifqs[i]; 457 sc->sc_q[i].rx.ifiq = ifp->if_iqs[i]; 458 459 #if NKSTAT > 0 460 vmx_kstat_txstats(sc, &sc->sc_q[i].tx, i); 461 vmx_kstat_rxstats(sc, &sc->sc_q[i].rx, i); 462 #endif 463 } 464 } 465 466 int 467 vmxnet3_dma_init(struct vmxnet3_softc *sc) 468 { 469 struct vmxnet3_driver_shared *ds; 470 struct vmxnet3_txq_shared *ts; 471 struct vmxnet3_rxq_shared *rs; 472 bus_addr_t ds_pa, qs_pa, mcast_pa; 473 int i, queue, qs_len, intr; 474 u_int major, minor, release_code, rev; 475 476 qs_len = sc->sc_nqueues * (sizeof *ts + sizeof *rs); 477 ts = vmxnet3_dma_allocmem(sc, qs_len, VMXNET3_DMADESC_ALIGN, &qs_pa); 478 if (ts == NULL) 479 return -1; 480 for (queue = 0; queue < sc->sc_nqueues; queue++) 481 sc->sc_q[queue].tx.ts = ts++; 482 rs = (void *)ts; 483 for (queue = 0; queue < sc->sc_nqueues; queue++) 484 sc->sc_q[queue].rx.rs = rs++; 485 486 for (queue = 0; queue < sc->sc_nqueues; queue++) { 487 intr = sc->sc_q[queue].intr; 488 489 if (vmxnet3_alloc_txring(sc, queue, intr)) 490 return -1; 491 if (vmxnet3_alloc_rxring(sc, queue, intr)) 492 return -1; 493 } 494 495 sc->sc_mcast = vmxnet3_dma_allocmem(sc, 682 * ETHER_ADDR_LEN, 32, &mcast_pa); 496 if (sc->sc_mcast == NULL) 497 return -1; 498 499 ds = vmxnet3_dma_allocmem(sc, sizeof *sc->sc_ds, 8, &ds_pa); 500 if (ds == NULL) 501 return -1; 502 sc->sc_ds = ds; 503 ds->magic = VMXNET3_REV1_MAGIC; 504 ds->version = VMXNET3_DRIVER_VERSION; 505 506 /* 507 * XXX FreeBSD version uses following values: 508 * (Does the device behavior depend on them?) 509 * 510 * major = __FreeBSD_version / 100000; 511 * minor = (__FreeBSD_version / 1000) % 100; 512 * release_code = (__FreeBSD_version / 100) % 10; 513 * rev = __FreeBSD_version % 100; 514 */ 515 major = 0; 516 minor = 0; 517 release_code = 0; 518 rev = 0; 519 #ifdef __LP64__ 520 ds->guest = release_code << 30 | rev << 22 | major << 14 | minor << 6 521 | VMXNET3_GOS_FREEBSD | VMXNET3_GOS_64BIT; 522 #else 523 ds->guest = release_code << 30 | rev << 22 | major << 14 | minor << 6 524 | VMXNET3_GOS_FREEBSD | VMXNET3_GOS_32BIT; 525 #endif 526 ds->vmxnet3_revision = 1; 527 ds->upt_version = 1; 528 ds->upt_features = UPT1_F_CSUM; 529 #if NVLAN > 0 530 ds->upt_features |= UPT1_F_VLAN; 531 #endif 532 ds->driver_data = ~0ULL; 533 ds->driver_data_len = 0; 534 ds->queue_shared = qs_pa; 535 ds->queue_shared_len = qs_len; 536 ds->mtu = VMXNET3_MAX_MTU; 537 ds->ntxqueue = sc->sc_nqueues; 538 ds->nrxqueue = sc->sc_nqueues; 539 ds->mcast_table = mcast_pa; 540 ds->automask = 1; 541 ds->nintr = 1 + (sc->sc_intrmap != NULL ? sc->sc_nqueues : 0); 542 ds->evintr = 0; 543 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL; 544 for (i = 0; i < ds->nintr; i++) 545 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE; 546 547 if (sc->sc_nqueues > 1) { 548 struct vmxnet3_upt1_rss_conf *rsscfg; 549 bus_addr_t rss_pa; 550 551 rsscfg = vmxnet3_dma_allocmem(sc, sizeof(*rsscfg), 8, &rss_pa); 552 553 rsscfg->hash_type = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 554 UPT1_RSS_HASH_TYPE_IPV4 | 555 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 556 UPT1_RSS_HASH_TYPE_IPV6; 557 rsscfg->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ; 558 rsscfg->hash_key_size = sizeof(rsscfg->hash_key); 559 stoeplitz_to_key(rsscfg->hash_key, sizeof(rsscfg->hash_key)); 560 561 rsscfg->ind_table_size = sizeof(rsscfg->ind_table); 562 for (i = 0; i < sizeof(rsscfg->ind_table); i++) 563 rsscfg->ind_table[i] = i % sc->sc_nqueues; 564 565 ds->upt_features |= UPT1_F_RSS; 566 ds->rss.version = 1; 567 ds->rss.len = sizeof(*rsscfg); 568 ds->rss.paddr = rss_pa; 569 570 sc->sc_rss = rsscfg; 571 } 572 573 WRITE_BAR1(sc, VMXNET3_BAR1_DSL, ds_pa); 574 WRITE_BAR1(sc, VMXNET3_BAR1_DSH, (u_int64_t)ds_pa >> 32); 575 return 0; 576 } 577 578 int 579 vmxnet3_alloc_txring(struct vmxnet3_softc *sc, int queue, int intr) 580 { 581 struct vmxnet3_txqueue *tq = &sc->sc_q[queue].tx; 582 struct vmxnet3_txq_shared *ts; 583 struct vmxnet3_txring *ring = &tq->cmd_ring; 584 struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; 585 int idx; 586 587 tq->queue = queue; 588 589 if (vmx_dmamem_alloc(sc, &ring->dmamem, 590 NTXDESC * sizeof(struct vmxnet3_txdesc), 512) != 0) 591 return -1; 592 ring->txd = VMX_DMA_KVA(&ring->dmamem); 593 if (vmx_dmamem_alloc(sc, &comp_ring->dmamem, 594 NTXCOMPDESC * sizeof(comp_ring->txcd[0]), 512) != 0) 595 return -1; 596 comp_ring->txcd = VMX_DMA_KVA(&comp_ring->dmamem); 597 598 for (idx = 0; idx < NTXDESC; idx++) { 599 if (bus_dmamap_create(sc->sc_dmat, MAXMCLBYTES, NTXSEGS, 600 VMXNET3_TX_LEN_M, 0, BUS_DMA_NOWAIT, &ring->dmap[idx])) 601 return -1; 602 } 603 604 ts = tq->ts; 605 bzero(ts, sizeof *ts); 606 ts->npending = 0; 607 ts->intr_threshold = 1; 608 ts->cmd_ring = VMX_DMA_DVA(&ring->dmamem); 609 ts->cmd_ring_len = NTXDESC; 610 ts->comp_ring = VMX_DMA_DVA(&comp_ring->dmamem); 611 ts->comp_ring_len = NTXCOMPDESC; 612 ts->driver_data = ~0ULL; 613 ts->driver_data_len = 0; 614 ts->intr_idx = intr; 615 ts->stopped = 1; 616 ts->error = 0; 617 return 0; 618 } 619 620 int 621 vmxnet3_alloc_rxring(struct vmxnet3_softc *sc, int queue, int intr) 622 { 623 struct vmxnet3_rxqueue *rq = &sc->sc_q[queue].rx; 624 struct vmxnet3_rxq_shared *rs; 625 struct vmxnet3_rxring *ring; 626 struct vmxnet3_comp_ring *comp_ring; 627 int i, idx; 628 629 for (i = 0; i < 2; i++) { 630 ring = &rq->cmd_ring[i]; 631 if (vmx_dmamem_alloc(sc, &ring->dmamem, 632 NRXDESC * sizeof(struct vmxnet3_rxdesc), 512) != 0) 633 return -1; 634 ring->rxd = VMX_DMA_KVA(&ring->dmamem); 635 } 636 comp_ring = &rq->comp_ring; 637 if (vmx_dmamem_alloc(sc, &comp_ring->dmamem, 638 NRXCOMPDESC * sizeof(comp_ring->rxcd[0]), 512) != 0) 639 return -1; 640 comp_ring->rxcd = VMX_DMA_KVA(&comp_ring->dmamem); 641 642 for (i = 0; i < 2; i++) { 643 ring = &rq->cmd_ring[i]; 644 ring->sc = sc; 645 ring->rid = i; 646 mtx_init(&ring->mtx, IPL_NET); 647 timeout_set(&ring->refill, vmxnet3_rxfill_tick, ring); 648 for (idx = 0; idx < NRXDESC; idx++) { 649 if (bus_dmamap_create(sc->sc_dmat, JUMBO_LEN, 1, 650 JUMBO_LEN, 0, BUS_DMA_NOWAIT, &ring->dmap[idx])) 651 return -1; 652 } 653 654 ring->rs = rq->rs; 655 ring->rxh = (i == 0) ? 656 VMXNET3_BAR0_RXH1(queue) : VMXNET3_BAR0_RXH2(queue); 657 } 658 659 rs = rq->rs; 660 bzero(rs, sizeof *rs); 661 rs->cmd_ring[0] = VMX_DMA_DVA(&rq->cmd_ring[0].dmamem); 662 rs->cmd_ring[1] = VMX_DMA_DVA(&rq->cmd_ring[1].dmamem); 663 rs->cmd_ring_len[0] = NRXDESC; 664 rs->cmd_ring_len[1] = NRXDESC; 665 rs->comp_ring = VMX_DMA_DVA(&comp_ring->dmamem); 666 rs->comp_ring_len = NRXCOMPDESC; 667 rs->driver_data = ~0ULL; 668 rs->driver_data_len = 0; 669 rs->intr_idx = intr; 670 rs->stopped = 1; 671 rs->error = 0; 672 return 0; 673 } 674 675 void 676 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq) 677 { 678 struct vmxnet3_txring *ring = &tq->cmd_ring; 679 struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; 680 681 ring->cons = ring->prod = 0; 682 ring->gen = VMX_TX_GEN; 683 comp_ring->next = 0; 684 comp_ring->gen = VMX_TXC_GEN; 685 memset(VMX_DMA_KVA(&ring->dmamem), 0, 686 VMX_DMA_LEN(&ring->dmamem)); 687 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 688 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_PREWRITE); 689 memset(VMX_DMA_KVA(&comp_ring->dmamem), 0, 690 VMX_DMA_LEN(&comp_ring->dmamem)); 691 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 692 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_PREREAD); 693 694 ifq_clr_oactive(tq->ifq); 695 } 696 697 void 698 vmxnet3_rxfill_tick(void *arg) 699 { 700 struct vmxnet3_rxring *ring = arg; 701 702 if (!mtx_enter_try(&ring->mtx)) 703 return; 704 705 vmxnet3_rxfill(ring); 706 mtx_leave(&ring->mtx); 707 } 708 709 void 710 vmxnet3_rxfill(struct vmxnet3_rxring *ring) 711 { 712 struct vmxnet3_softc *sc = ring->sc; 713 struct vmxnet3_rxdesc *rxd; 714 struct mbuf *m; 715 bus_dmamap_t map; 716 u_int slots; 717 unsigned int prod; 718 uint32_t rgen; 719 uint32_t type = htole32(VMXNET3_BTYPE_HEAD << VMXNET3_RX_BTYPE_S); 720 721 /* Second ring just contains packet bodies. */ 722 if (ring->rid == 1) 723 type = htole32(VMXNET3_BTYPE_BODY << VMXNET3_RX_BTYPE_S); 724 725 MUTEX_ASSERT_LOCKED(&ring->mtx); 726 727 slots = if_rxr_get(&ring->rxr, NRXDESC); 728 if (slots == 0) 729 return; 730 731 prod = ring->fill; 732 rgen = ring->gen; 733 734 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 735 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_POSTWRITE); 736 737 do { 738 KASSERT(ring->m[prod] == NULL); 739 740 m = MCLGETL(NULL, M_DONTWAIT, JUMBO_LEN); 741 if (m == NULL) 742 break; 743 744 m->m_pkthdr.len = m->m_len = JUMBO_LEN; 745 m_adj(m, ETHER_ALIGN); 746 747 map = ring->dmap[prod]; 748 if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, BUS_DMA_NOWAIT)) 749 panic("load mbuf"); 750 751 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 752 BUS_DMASYNC_PREREAD); 753 754 ring->m[prod] = m; 755 756 rxd = &ring->rxd[prod]; 757 rxd->rx_addr = htole64(DMAADDR(map)); 758 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 759 0, VMX_DMA_LEN(&ring->dmamem), 760 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE); 761 rxd->rx_word2 = (htole32(m->m_pkthdr.len & VMXNET3_RX_LEN_M) << 762 VMXNET3_RX_LEN_S) | type | rgen; 763 764 if (++prod == NRXDESC) { 765 prod = 0; 766 rgen ^= VMX_RX_GEN; 767 } 768 } while (--slots > 0); 769 770 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 771 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_PREWRITE); 772 773 if_rxr_put(&ring->rxr, slots); 774 775 ring->fill = prod; 776 ring->gen = rgen; 777 778 if (if_rxr_inuse(&ring->rxr) == 0) 779 timeout_add(&ring->refill, 1); 780 781 if (ring->rs->update_rxhead) 782 WRITE_BAR0(sc, ring->rxh, prod); 783 } 784 785 void 786 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq) 787 { 788 struct vmxnet3_rxring *ring; 789 struct vmxnet3_comp_ring *comp_ring; 790 int i; 791 792 for (i = 0; i < 2; i++) { 793 ring = &rq->cmd_ring[i]; 794 if_rxr_init(&ring->rxr, 2, NRXDESC - 1); 795 ring->fill = 0; 796 ring->gen = VMX_RX_GEN; 797 798 memset(VMX_DMA_KVA(&ring->dmamem), 0, 799 VMX_DMA_LEN(&ring->dmamem)); 800 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 801 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_PREWRITE); 802 803 mtx_enter(&ring->mtx); 804 vmxnet3_rxfill(ring); 805 mtx_leave(&ring->mtx); 806 } 807 808 comp_ring = &rq->comp_ring; 809 comp_ring->next = 0; 810 comp_ring->gen = VMX_RXC_GEN; 811 comp_ring->sendmp = NULL; 812 comp_ring->lastmp = NULL; 813 814 memset(VMX_DMA_KVA(&comp_ring->dmamem), 0, 815 VMX_DMA_LEN(&comp_ring->dmamem)); 816 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 817 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_PREREAD); 818 } 819 820 void 821 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq) 822 { 823 struct vmxnet3_txring *ring = &tq->cmd_ring; 824 struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; 825 struct ifqueue *ifq = tq->ifq; 826 int idx; 827 828 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 829 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_POSTREAD); 830 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 831 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_POSTWRITE); 832 833 for (idx = 0; idx < NTXDESC; idx++) { 834 if (ring->m[idx]) { 835 bus_dmamap_unload(sc->sc_dmat, ring->dmap[idx]); 836 m_freem(ring->m[idx]); 837 ring->m[idx] = NULL; 838 } 839 } 840 841 ifq_purge(ifq); 842 ifq_clr_oactive(ifq); 843 } 844 845 void 846 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq) 847 { 848 struct vmxnet3_rxring *ring; 849 struct vmxnet3_comp_ring *comp_ring = &rq->comp_ring; 850 int i, idx; 851 852 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 853 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_POSTREAD); 854 855 for (i = 0; i < 2; i++) { 856 ring = &rq->cmd_ring[i]; 857 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 858 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_POSTWRITE); 859 timeout_del(&ring->refill); 860 for (idx = 0; idx < NRXDESC; idx++) { 861 struct mbuf *m = ring->m[idx]; 862 if (m == NULL) 863 continue; 864 865 ring->m[idx] = NULL; 866 m_freem(m); 867 bus_dmamap_unload(sc->sc_dmat, ring->dmap[idx]); 868 } 869 } 870 } 871 872 void 873 vmxnet3_link_state(struct vmxnet3_softc *sc) 874 { 875 struct ifnet *ifp = &sc->sc_arpcom.ac_if; 876 u_int x, link, speed; 877 878 WRITE_CMD(sc, VMXNET3_CMD_GET_LINK); 879 x = READ_BAR1(sc, VMXNET3_BAR1_CMD); 880 speed = x >> 16; 881 if (x & 1) { 882 ifp->if_baudrate = IF_Mbps(speed); 883 link = LINK_STATE_UP; 884 } else 885 link = LINK_STATE_DOWN; 886 887 if (ifp->if_link_state != link) { 888 ifp->if_link_state = link; 889 if_link_state_change(ifp); 890 } 891 } 892 893 static inline void 894 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq) 895 { 896 WRITE_BAR0(sc, VMXNET3_BAR0_IMASK(irq), 0); 897 } 898 899 static inline void 900 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq) 901 { 902 WRITE_BAR0(sc, VMXNET3_BAR0_IMASK(irq), 1); 903 } 904 905 void 906 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc) 907 { 908 int i; 909 910 sc->sc_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL; 911 vmxnet3_enable_intr(sc, 0); 912 if (sc->sc_intrmap) { 913 for (i = 0; i < sc->sc_nqueues; i++) 914 vmxnet3_enable_intr(sc, sc->sc_q[i].intr); 915 } 916 } 917 918 void 919 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc) 920 { 921 int i; 922 923 sc->sc_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL; 924 vmxnet3_disable_intr(sc, 0); 925 if (sc->sc_intrmap) { 926 for (i = 0; i < sc->sc_nqueues; i++) 927 vmxnet3_disable_intr(sc, sc->sc_q[i].intr); 928 } 929 } 930 931 int 932 vmxnet3_intr_intx(void *arg) 933 { 934 struct vmxnet3_softc *sc = arg; 935 936 if (READ_BAR1(sc, VMXNET3_BAR1_INTR) == 0) 937 return 0; 938 939 return (vmxnet3_intr(sc)); 940 } 941 942 int 943 vmxnet3_intr(void *arg) 944 { 945 struct vmxnet3_softc *sc = arg; 946 struct ifnet *ifp = &sc->sc_arpcom.ac_if; 947 948 if (sc->sc_ds->event) { 949 KERNEL_LOCK(); 950 vmxnet3_evintr(sc); 951 KERNEL_UNLOCK(); 952 } 953 954 if (ifp->if_flags & IFF_RUNNING) { 955 vmxnet3_rxintr(sc, &sc->sc_q[0].rx); 956 vmxnet3_txintr(sc, &sc->sc_q[0].tx); 957 vmxnet3_enable_intr(sc, 0); 958 } 959 960 return 1; 961 } 962 963 int 964 vmxnet3_intr_event(void *arg) 965 { 966 struct vmxnet3_softc *sc = arg; 967 968 if (sc->sc_ds->event) { 969 KERNEL_LOCK(); 970 vmxnet3_evintr(sc); 971 KERNEL_UNLOCK(); 972 } 973 974 vmxnet3_enable_intr(sc, 0); 975 return 1; 976 } 977 978 int 979 vmxnet3_intr_queue(void *arg) 980 { 981 struct vmxnet3_queue *q = arg; 982 983 vmxnet3_rxintr(q->sc, &q->rx); 984 vmxnet3_txintr(q->sc, &q->tx); 985 vmxnet3_enable_intr(q->sc, q->intr); 986 987 return 1; 988 } 989 990 void 991 vmxnet3_evintr(struct vmxnet3_softc *sc) 992 { 993 struct ifnet *ifp = &sc->sc_arpcom.ac_if; 994 u_int event = sc->sc_ds->event; 995 struct vmxnet3_txq_shared *ts; 996 struct vmxnet3_rxq_shared *rs; 997 998 /* Clear events. */ 999 WRITE_BAR1(sc, VMXNET3_BAR1_EVENT, event); 1000 1001 /* Link state change? */ 1002 if (event & VMXNET3_EVENT_LINK) 1003 vmxnet3_link_state(sc); 1004 1005 /* Queue error? */ 1006 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) { 1007 WRITE_CMD(sc, VMXNET3_CMD_GET_STATUS); 1008 1009 ts = sc->sc_q[0].tx.ts; 1010 if (ts->stopped) 1011 printf("%s: TX error 0x%x\n", ifp->if_xname, ts->error); 1012 rs = sc->sc_q[0].rx.rs; 1013 if (rs->stopped) 1014 printf("%s: RX error 0x%x\n", ifp->if_xname, rs->error); 1015 vmxnet3_init(sc); 1016 } 1017 1018 if (event & VMXNET3_EVENT_DIC) 1019 printf("%s: device implementation change event\n", 1020 ifp->if_xname); 1021 if (event & VMXNET3_EVENT_DEBUG) 1022 printf("%s: debug event\n", ifp->if_xname); 1023 } 1024 1025 void 1026 vmxnet3_txintr(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq) 1027 { 1028 struct ifqueue *ifq = tq->ifq; 1029 struct vmxnet3_txring *ring = &tq->cmd_ring; 1030 struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; 1031 struct vmxnet3_txcompdesc *txcd; 1032 bus_dmamap_t map; 1033 struct mbuf *m; 1034 u_int prod, cons, next; 1035 uint32_t rgen; 1036 1037 prod = ring->prod; 1038 cons = ring->cons; 1039 1040 if (cons == prod) 1041 return; 1042 1043 next = comp_ring->next; 1044 rgen = comp_ring->gen; 1045 1046 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 1047 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_POSTREAD); 1048 1049 do { 1050 txcd = &comp_ring->txcd[next]; 1051 if ((txcd->txc_word3 & VMX_TXC_GEN) != rgen) 1052 break; 1053 1054 if (++next == NTXCOMPDESC) { 1055 next = 0; 1056 rgen ^= VMX_TXC_GEN; 1057 } 1058 1059 m = ring->m[cons]; 1060 ring->m[cons] = NULL; 1061 1062 KASSERT(m != NULL); 1063 1064 map = ring->dmap[cons]; 1065 bus_dmamap_unload(sc->sc_dmat, map); 1066 m_freem(m); 1067 1068 cons = (letoh32(txcd->txc_word0) >> VMXNET3_TXC_EOPIDX_S) & 1069 VMXNET3_TXC_EOPIDX_M; 1070 cons++; 1071 cons %= NTXDESC; 1072 } while (cons != prod); 1073 1074 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 1075 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_PREREAD); 1076 1077 comp_ring->next = next; 1078 comp_ring->gen = rgen; 1079 ring->cons = cons; 1080 1081 if (ifq_is_oactive(ifq)) 1082 ifq_restart(ifq); 1083 } 1084 1085 void 1086 vmxnet3_rxintr(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq) 1087 { 1088 struct ifnet *ifp = &sc->sc_arpcom.ac_if; 1089 struct vmxnet3_comp_ring *comp_ring = &rq->comp_ring; 1090 struct vmxnet3_rxring *ring; 1091 struct vmxnet3_rxcompdesc *rxcd; 1092 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 1093 struct mbuf *m; 1094 bus_dmamap_t map; 1095 unsigned int idx; 1096 unsigned int next, rgen; 1097 unsigned int rid, done[2] = {0, 0}; 1098 1099 next = comp_ring->next; 1100 rgen = comp_ring->gen; 1101 1102 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 1103 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_POSTREAD); 1104 1105 for (;;) { 1106 rxcd = &comp_ring->rxcd[next]; 1107 if ((rxcd->rxc_word3 & VMX_RXC_GEN) != rgen) 1108 break; 1109 1110 if (++next == NRXCOMPDESC) { 1111 next = 0; 1112 rgen ^= VMX_RXC_GEN; 1113 } 1114 1115 idx = letoh32((rxcd->rxc_word0 >> VMXNET3_RXC_IDX_S) & 1116 VMXNET3_RXC_IDX_M); 1117 1118 if (letoh32((rxcd->rxc_word0 >> VMXNET3_RXC_QID_S) & 1119 VMXNET3_RXC_QID_M) < sc->sc_nqueues) 1120 rid = 0; 1121 else 1122 rid = 1; 1123 1124 ring = &rq->cmd_ring[rid]; 1125 1126 m = ring->m[idx]; 1127 KASSERT(m != NULL); 1128 ring->m[idx] = NULL; 1129 1130 map = ring->dmap[idx]; 1131 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 1132 BUS_DMASYNC_POSTREAD); 1133 bus_dmamap_unload(sc->sc_dmat, map); 1134 1135 done[rid]++; 1136 1137 /* 1138 * A receive descriptor of type 4 which is flagged as start of 1139 * packet, contains the number of TCP segment of an LRO packet. 1140 */ 1141 if (letoh32((rxcd->rxc_word3 & VMXNET3_RXC_TYPE_M) >> 1142 VMXNET3_RXC_TYPE_S) == 4 && 1143 ISSET(rxcd->rxc_word0, VMXNET3_RXC_SOP)) { 1144 m->m_pkthdr.ph_mss = letoh32(rxcd->rxc_word1 & 1145 VMXNET3_RXC_SEG_CNT_M); 1146 } 1147 1148 m->m_len = letoh32((rxcd->rxc_word2 >> VMXNET3_RXC_LEN_S) & 1149 VMXNET3_RXC_LEN_M); 1150 1151 if (comp_ring->sendmp == NULL) { 1152 comp_ring->sendmp = comp_ring->lastmp = m; 1153 comp_ring->sendmp->m_pkthdr.len = 0; 1154 } else { 1155 CLR(m->m_flags, M_PKTHDR); 1156 comp_ring->lastmp->m_next = m; 1157 comp_ring->lastmp = m; 1158 } 1159 comp_ring->sendmp->m_pkthdr.len += m->m_len; 1160 1161 if (!ISSET(rxcd->rxc_word0, VMXNET3_RXC_EOP)) 1162 continue; 1163 1164 /* 1165 * End of Packet 1166 */ 1167 1168 if (letoh32(rxcd->rxc_word2 & VMXNET3_RXC_ERROR)) { 1169 ifp->if_ierrors++; 1170 m_freem(comp_ring->sendmp); 1171 comp_ring->sendmp = comp_ring->lastmp = NULL; 1172 continue; 1173 } 1174 1175 if (comp_ring->sendmp->m_pkthdr.len < VMXNET3_MIN_MTU) { 1176 m_freem(comp_ring->sendmp); 1177 comp_ring->sendmp = comp_ring->lastmp = NULL; 1178 continue; 1179 } 1180 1181 if (((letoh32(rxcd->rxc_word0) >> VMXNET3_RXC_RSSTYPE_S) & 1182 VMXNET3_RXC_RSSTYPE_M) != VMXNET3_RXC_RSSTYPE_NONE) { 1183 comp_ring->sendmp->m_pkthdr.ph_flowid = 1184 letoh32(rxcd->rxc_word1); 1185 SET(comp_ring->sendmp->m_pkthdr.csum_flags, M_FLOWID); 1186 } 1187 1188 vmxnet3_rx_offload(rxcd, comp_ring->sendmp); 1189 ml_enqueue(&ml, comp_ring->sendmp); 1190 comp_ring->sendmp = comp_ring->lastmp = NULL; 1191 } 1192 1193 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&comp_ring->dmamem), 1194 0, VMX_DMA_LEN(&comp_ring->dmamem), BUS_DMASYNC_PREREAD); 1195 1196 comp_ring->next = next; 1197 comp_ring->gen = rgen; 1198 1199 for (int i = 0; i < 2; i++) { 1200 if (done[i] == 0) 1201 continue; 1202 1203 ring = &rq->cmd_ring[i]; 1204 1205 if (ifiq_input(rq->ifiq, &ml)) 1206 if_rxr_livelocked(&ring->rxr); 1207 1208 mtx_enter(&ring->mtx); 1209 if_rxr_put(&ring->rxr, done[i]); 1210 vmxnet3_rxfill(ring); 1211 mtx_leave(&ring->mtx); 1212 } 1213 } 1214 1215 void 1216 vmxnet3_iff(struct vmxnet3_softc *sc) 1217 { 1218 struct ifnet *ifp = &sc->sc_arpcom.ac_if; 1219 struct arpcom *ac = &sc->sc_arpcom; 1220 struct vmxnet3_driver_shared *ds = sc->sc_ds; 1221 struct ether_multi *enm; 1222 struct ether_multistep step; 1223 u_int mode; 1224 u_int8_t *p; 1225 1226 ds->mcast_tablelen = 0; 1227 CLR(ifp->if_flags, IFF_ALLMULTI); 1228 1229 /* 1230 * Always accept broadcast frames. 1231 * Always accept frames destined to our station address. 1232 */ 1233 mode = VMXNET3_RXMODE_BCAST | VMXNET3_RXMODE_UCAST; 1234 1235 if (ISSET(ifp->if_flags, IFF_PROMISC) || ac->ac_multirangecnt > 0 || 1236 ac->ac_multicnt > 682) { 1237 SET(ifp->if_flags, IFF_ALLMULTI); 1238 SET(mode, (VMXNET3_RXMODE_ALLMULTI | VMXNET3_RXMODE_MCAST)); 1239 if (ifp->if_flags & IFF_PROMISC) 1240 SET(mode, VMXNET3_RXMODE_PROMISC); 1241 } else { 1242 p = sc->sc_mcast; 1243 ETHER_FIRST_MULTI(step, ac, enm); 1244 while (enm != NULL) { 1245 bcopy(enm->enm_addrlo, p, ETHER_ADDR_LEN); 1246 1247 p += ETHER_ADDR_LEN; 1248 1249 ETHER_NEXT_MULTI(step, enm); 1250 } 1251 1252 if (ac->ac_multicnt > 0) { 1253 SET(mode, VMXNET3_RXMODE_MCAST); 1254 ds->mcast_tablelen = p - sc->sc_mcast; 1255 } 1256 } 1257 1258 WRITE_CMD(sc, VMXNET3_CMD_SET_FILTER); 1259 ds->rxmode = mode; 1260 WRITE_CMD(sc, VMXNET3_CMD_SET_RXMODE); 1261 } 1262 1263 1264 void 1265 vmxnet3_rx_offload(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m) 1266 { 1267 uint32_t pkts; 1268 1269 /* 1270 * VLAN Offload 1271 */ 1272 1273 #if NVLAN > 0 1274 if (ISSET(rxcd->rxc_word2, VMXNET3_RXC_VLAN)) { 1275 SET(m->m_flags, M_VLANTAG); 1276 m->m_pkthdr.ether_vtag = letoh32((rxcd->rxc_word2 >> 1277 VMXNET3_RXC_VLANTAG_S) & VMXNET3_RXC_VLANTAG_M); 1278 } 1279 #endif 1280 1281 /* 1282 * Checksum Offload 1283 */ 1284 1285 if (ISSET(rxcd->rxc_word0, VMXNET3_RXC_NOCSUM)) 1286 return; 1287 1288 if (ISSET(rxcd->rxc_word3, VMXNET3_RXC_IPV4) && 1289 ISSET(rxcd->rxc_word3, VMXNET3_RXC_IPSUM_OK)) 1290 SET(m->m_pkthdr.csum_flags, M_IPV4_CSUM_IN_OK); 1291 1292 if (ISSET(rxcd->rxc_word3, VMXNET3_RXC_FRAGMENT)) 1293 return; 1294 1295 if (ISSET(rxcd->rxc_word3, VMXNET3_RXC_CSUM_OK)) { 1296 if (ISSET(rxcd->rxc_word3, VMXNET3_RXC_TCP)) 1297 SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_OK); 1298 else if (ISSET(rxcd->rxc_word3, VMXNET3_RXC_UDP)) 1299 SET(m->m_pkthdr.csum_flags, M_UDP_CSUM_IN_OK); 1300 } 1301 1302 /* 1303 * TCP Large Receive Offload 1304 */ 1305 1306 pkts = m->m_pkthdr.ph_mss; 1307 m->m_pkthdr.ph_mss = 0; 1308 1309 if (pkts > 1) { 1310 struct ether_extracted ext; 1311 uint32_t paylen; 1312 1313 ether_extract_headers(m, &ext); 1314 1315 paylen = ext.iplen; 1316 if (ext.ip4 || ext.ip6) 1317 paylen -= ext.iphlen; 1318 1319 if (ext.tcp) { 1320 paylen -= ext.tcphlen; 1321 tcpstat_inc(tcps_inhwlro); 1322 tcpstat_add(tcps_inpktlro, pkts); 1323 } else { 1324 tcpstat_inc(tcps_inbadlro); 1325 } 1326 1327 /* 1328 * If we gonna forward this packet, we have to mark it as TSO, 1329 * set a correct mss, and recalculate the TCP checksum. 1330 */ 1331 if (ext.tcp && paylen >= pkts) { 1332 SET(m->m_pkthdr.csum_flags, M_TCP_TSO); 1333 m->m_pkthdr.ph_mss = paylen / pkts; 1334 } 1335 if (ext.tcp && 1336 ISSET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_OK)) { 1337 SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_OUT); 1338 } 1339 } 1340 } 1341 1342 void 1343 vmxnet3_stop(struct ifnet *ifp) 1344 { 1345 struct vmxnet3_softc *sc = ifp->if_softc; 1346 int queue; 1347 1348 ifp->if_flags &= ~IFF_RUNNING; 1349 ifp->if_timer = 0; 1350 1351 vmxnet3_disable_all_intrs(sc); 1352 1353 WRITE_CMD(sc, VMXNET3_CMD_DISABLE); 1354 1355 if (sc->sc_intrmap != NULL) { 1356 for (queue = 0; queue < sc->sc_nqueues; queue++) 1357 intr_barrier(sc->sc_q[queue].ih); 1358 } else 1359 intr_barrier(sc->sc_ih); 1360 1361 for (queue = 0; queue < sc->sc_nqueues; queue++) 1362 vmxnet3_txstop(sc, &sc->sc_q[queue].tx); 1363 for (queue = 0; queue < sc->sc_nqueues; queue++) 1364 vmxnet3_rxstop(sc, &sc->sc_q[queue].rx); 1365 } 1366 1367 void 1368 vmxnet3_reset(struct vmxnet3_softc *sc) 1369 { 1370 WRITE_CMD(sc, VMXNET3_CMD_RESET); 1371 } 1372 1373 int 1374 vmxnet3_init(struct vmxnet3_softc *sc) 1375 { 1376 struct ifnet *ifp = &sc->sc_arpcom.ac_if; 1377 int queue; 1378 1379 /* 1380 * Cancel pending I/O and free all RX/TX buffers. 1381 */ 1382 vmxnet3_stop(ifp); 1383 1384 #if 0 1385 /* Put controller into known state. */ 1386 vmxnet3_reset(sc); 1387 #endif 1388 1389 for (queue = 0; queue < sc->sc_nqueues; queue++) 1390 vmxnet3_txinit(sc, &sc->sc_q[queue].tx); 1391 for (queue = 0; queue < sc->sc_nqueues; queue++) 1392 vmxnet3_rxinit(sc, &sc->sc_q[queue].rx); 1393 1394 for (queue = 0; queue < sc->sc_nqueues; queue++) { 1395 WRITE_BAR0(sc, VMXNET3_BAR0_RXH1(queue), 0); 1396 WRITE_BAR0(sc, VMXNET3_BAR0_RXH2(queue), 0); 1397 } 1398 1399 WRITE_CMD(sc, VMXNET3_CMD_ENABLE); 1400 if (READ_BAR1(sc, VMXNET3_BAR1_CMD)) { 1401 printf("%s: failed to initialize\n", ifp->if_xname); 1402 vmxnet3_stop(ifp); 1403 return EIO; 1404 } 1405 1406 /* TCP Large Receive Offload */ 1407 if (ISSET(ifp->if_xflags, IFXF_LRO)) 1408 SET(sc->sc_ds->upt_features, UPT1_F_LRO); 1409 else 1410 CLR(sc->sc_ds->upt_features, UPT1_F_LRO); 1411 WRITE_CMD(sc, VMXNET3_CMD_SET_FEATURE); 1412 1413 /* Program promiscuous mode and multicast filters. */ 1414 vmxnet3_iff(sc); 1415 1416 vmxnet3_enable_all_intrs(sc); 1417 1418 vmxnet3_link_state(sc); 1419 1420 ifp->if_flags |= IFF_RUNNING; 1421 1422 return 0; 1423 } 1424 1425 static int 1426 vmx_rxr_info(struct vmxnet3_softc *sc, struct if_rxrinfo *ifri) 1427 { 1428 struct if_rxring_info *ifrs, *ifr; 1429 int error; 1430 unsigned int i; 1431 1432 ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), 1433 M_TEMP, M_WAITOK|M_ZERO|M_CANFAIL); 1434 if (ifrs == NULL) 1435 return (ENOMEM); 1436 1437 for (i = 0; i < sc->sc_nqueues; i++) { 1438 struct if_rxring *rxr = &sc->sc_q[i].rx.cmd_ring[0].rxr; 1439 ifr = &ifrs[i]; 1440 1441 ifr->ifr_size = JUMBO_LEN; 1442 snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i); 1443 ifr->ifr_info = *rxr; 1444 } 1445 1446 error = if_rxr_info_ioctl(ifri, i, ifrs); 1447 1448 free(ifrs, M_TEMP, i * sizeof(*ifrs)); 1449 1450 return (error); 1451 } 1452 1453 int 1454 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1455 { 1456 struct vmxnet3_softc *sc = ifp->if_softc; 1457 struct ifreq *ifr = (struct ifreq *)data; 1458 int error = 0, s; 1459 1460 s = splnet(); 1461 1462 switch (cmd) { 1463 case SIOCSIFADDR: 1464 ifp->if_flags |= IFF_UP; 1465 if ((ifp->if_flags & IFF_RUNNING) == 0) 1466 error = vmxnet3_init(sc); 1467 break; 1468 case SIOCSIFFLAGS: 1469 if (ifp->if_flags & IFF_UP) { 1470 if (ifp->if_flags & IFF_RUNNING) 1471 error = ENETRESET; 1472 else 1473 error = vmxnet3_init(sc); 1474 } else { 1475 if (ifp->if_flags & IFF_RUNNING) 1476 vmxnet3_stop(ifp); 1477 } 1478 break; 1479 case SIOCSIFMEDIA: 1480 case SIOCGIFMEDIA: 1481 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 1482 break; 1483 case SIOCGIFRXR: 1484 error = vmx_rxr_info(sc, (struct if_rxrinfo *)ifr->ifr_data); 1485 break; 1486 default: 1487 error = ether_ioctl(ifp, &sc->sc_arpcom, cmd, data); 1488 } 1489 1490 if (error == ENETRESET) { 1491 if (ifp->if_flags & IFF_RUNNING) 1492 vmxnet3_iff(sc); 1493 error = 0; 1494 } 1495 1496 splx(s); 1497 return error; 1498 } 1499 1500 static inline int 1501 vmx_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m) 1502 { 1503 int error; 1504 1505 error = bus_dmamap_load_mbuf(dmat, map, m, 1506 BUS_DMA_STREAMING | BUS_DMA_NOWAIT); 1507 if (error != EFBIG) 1508 return (error); 1509 1510 error = m_defrag(m, M_DONTWAIT); 1511 if (error != 0) 1512 return (error); 1513 1514 return (bus_dmamap_load_mbuf(dmat, map, m, 1515 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)); 1516 } 1517 1518 void 1519 vmxnet3_tx_offload(struct vmxnet3_txdesc *sop, struct mbuf *m) 1520 { 1521 struct ether_extracted ext; 1522 uint32_t offset = 0; 1523 uint32_t hdrlen; 1524 1525 /* 1526 * VLAN Offload 1527 */ 1528 1529 #if NVLAN > 0 1530 if (ISSET(m->m_flags, M_VLANTAG)) { 1531 sop->tx_word3 |= htole32(VMXNET3_TX_VTAG_MODE); 1532 sop->tx_word3 |= htole32((m->m_pkthdr.ether_vtag & 1533 VMXNET3_TX_VLANTAG_M) << VMXNET3_TX_VLANTAG_S); 1534 } 1535 #endif 1536 1537 /* 1538 * Checksum Offload 1539 */ 1540 1541 if (!ISSET(m->m_pkthdr.csum_flags, M_TCP_CSUM_OUT) && 1542 !ISSET(m->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) 1543 return; 1544 1545 ether_extract_headers(m, &ext); 1546 1547 hdrlen = sizeof(*ext.eh); 1548 if (ext.evh) 1549 hdrlen = sizeof(*ext.evh); 1550 1551 if (ext.ip4 || ext.ip6) 1552 hdrlen += ext.iphlen; 1553 1554 if (ext.tcp) 1555 offset = hdrlen + offsetof(struct tcphdr, th_sum); 1556 else if (ext.udp) 1557 offset = hdrlen + offsetof(struct udphdr, uh_sum); 1558 else 1559 return; 1560 1561 if (!ISSET(m->m_pkthdr.csum_flags, M_TCP_TSO)) { 1562 hdrlen &= VMXNET3_TX_HLEN_M; 1563 offset &= VMXNET3_TX_OP_M; 1564 1565 sop->tx_word3 |= htole32(VMXNET3_OM_CSUM << VMXNET3_TX_OM_S); 1566 sop->tx_word3 |= htole32(hdrlen << VMXNET3_TX_HLEN_S); 1567 sop->tx_word2 |= htole32(offset << VMXNET3_TX_OP_S); 1568 1569 return; 1570 } 1571 1572 /* 1573 * TCP Segmentation Offload 1574 */ 1575 1576 if (ext.tcp == NULL || m->m_pkthdr.ph_mss == 0) { 1577 tcpstat_inc(tcps_outbadtso); 1578 return; 1579 } 1580 1581 if (ext.ip4) 1582 ext.ip4->ip_sum = 0; 1583 1584 hdrlen += ext.tcphlen; 1585 hdrlen &= VMXNET3_TX_HLEN_M; 1586 1587 sop->tx_word3 |= htole32(VMXNET3_OM_TSO << VMXNET3_TX_OM_S); 1588 sop->tx_word3 |= htole32(hdrlen << VMXNET3_TX_HLEN_S); 1589 sop->tx_word2 |= htole32(m->m_pkthdr.ph_mss << VMXNET3_TX_OP_S); 1590 1591 tcpstat_add(tcps_outpkttso, (m->m_pkthdr.len - hdrlen + 1592 m->m_pkthdr.ph_mss - 1) / m->m_pkthdr.ph_mss); 1593 } 1594 1595 void 1596 vmxnet3_start(struct ifqueue *ifq) 1597 { 1598 struct ifnet *ifp = ifq->ifq_if; 1599 struct vmxnet3_softc *sc = ifp->if_softc; 1600 struct vmxnet3_txqueue *tq = ifq->ifq_softc; 1601 struct vmxnet3_txring *ring = &tq->cmd_ring; 1602 struct vmxnet3_txdesc *txd, *sop; 1603 bus_dmamap_t map; 1604 unsigned int prod, free, i; 1605 unsigned int post = 0; 1606 uint32_t rgen, gen; 1607 1608 struct mbuf *m; 1609 1610 free = ring->cons; 1611 prod = ring->prod; 1612 if (free <= prod) 1613 free += NTXDESC; 1614 free -= prod; 1615 1616 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 1617 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_POSTWRITE); 1618 1619 rgen = ring->gen; 1620 1621 for (;;) { 1622 int hdrlen; 1623 1624 if (free <= NTXSEGS) 1625 break; 1626 1627 m = ifq_dequeue(ifq); 1628 if (m == NULL) 1629 break; 1630 1631 /* 1632 * Headers for Ether, IP, TCP including options must lay in 1633 * first mbuf to support TSO. Usually our stack gets that 1634 * right. To avoid packet parsing here, make a rough estimate 1635 * for simple IPv4. Cases seen in the wild contain only ether 1636 * header in separate mbuf. To support IPv6 with TCP options, 1637 * move as much as possible into first mbuf. Realloc mbuf 1638 * before bus dma load. 1639 */ 1640 hdrlen = sizeof(struct ether_header) + sizeof(struct ip) + 1641 sizeof(struct tcphdr); 1642 if (ISSET(m->m_pkthdr.csum_flags, M_TCP_TSO) && 1643 m->m_len < hdrlen && hdrlen <= m->m_pkthdr.len) { 1644 hdrlen = MHLEN; 1645 /* m_pullup preserves alignment, reserve space */ 1646 hdrlen -= mtod(m, unsigned long) & (sizeof(long) - 1); 1647 if (hdrlen > m->m_pkthdr.len) 1648 hdrlen = m->m_pkthdr.len; 1649 if ((m = m_pullup(m, hdrlen)) == NULL) { 1650 ifq->ifq_errors++; 1651 continue; 1652 } 1653 } 1654 1655 map = ring->dmap[prod]; 1656 1657 if (vmx_load_mbuf(sc->sc_dmat, map, m) != 0) { 1658 ifq->ifq_errors++; 1659 m_freem(m); 1660 continue; 1661 } 1662 1663 #if NBPFILTER > 0 1664 if (ifp->if_bpf) 1665 bpf_mtap_ether(ifp->if_bpf, m, BPF_DIRECTION_OUT); 1666 #endif 1667 1668 ring->m[prod] = m; 1669 1670 bus_dmamap_sync(sc->sc_dmat, map, 0, 1671 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1672 1673 free -= map->dm_nsegs; 1674 /* set oactive here since txintr may be triggered in parallel */ 1675 if (free <= NTXSEGS) 1676 ifq_set_oactive(ifq); 1677 1678 gen = rgen ^ VMX_TX_GEN; 1679 sop = &ring->txd[prod]; 1680 for (i = 0; i < map->dm_nsegs; i++) { 1681 txd = &ring->txd[prod]; 1682 txd->tx_addr = htole64(map->dm_segs[i].ds_addr); 1683 txd->tx_word2 = htole32(map->dm_segs[i].ds_len << 1684 VMXNET3_TX_LEN_S) | gen; 1685 txd->tx_word3 = 0; 1686 1687 if (++prod == NTXDESC) { 1688 prod = 0; 1689 rgen ^= VMX_TX_GEN; 1690 } 1691 1692 gen = rgen; 1693 } 1694 txd->tx_word3 = htole32(VMXNET3_TX_EOP | VMXNET3_TX_COMPREQ); 1695 1696 vmxnet3_tx_offload(sop, m); 1697 1698 ring->prod = prod; 1699 /* Change the ownership by flipping the "generation" bit */ 1700 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 1701 0, VMX_DMA_LEN(&ring->dmamem), 1702 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE); 1703 sop->tx_word2 ^= VMX_TX_GEN; 1704 1705 post = 1; 1706 } 1707 1708 bus_dmamap_sync(sc->sc_dmat, VMX_DMA_MAP(&ring->dmamem), 1709 0, VMX_DMA_LEN(&ring->dmamem), BUS_DMASYNC_PREWRITE); 1710 1711 if (!post) 1712 return; 1713 1714 ring->gen = rgen; 1715 1716 WRITE_BAR0(sc, VMXNET3_BAR0_TXH(tq->queue), prod); 1717 } 1718 1719 void 1720 vmxnet3_watchdog(struct ifnet *ifp) 1721 { 1722 struct vmxnet3_softc *sc = ifp->if_softc; 1723 int s; 1724 1725 printf("%s: device timeout\n", ifp->if_xname); 1726 s = splnet(); 1727 vmxnet3_init(sc); 1728 splx(s); 1729 } 1730 1731 void 1732 vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1733 { 1734 struct vmxnet3_softc *sc = ifp->if_softc; 1735 1736 vmxnet3_link_state(sc); 1737 1738 ifmr->ifm_status = IFM_AVALID; 1739 ifmr->ifm_active = IFM_ETHER; 1740 1741 if (ifp->if_link_state != LINK_STATE_UP) 1742 return; 1743 1744 ifmr->ifm_status |= IFM_ACTIVE; 1745 1746 if (ifp->if_baudrate >= IF_Gbps(10)) 1747 ifmr->ifm_active |= IFM_10G_T; 1748 } 1749 1750 int 1751 vmxnet3_media_change(struct ifnet *ifp) 1752 { 1753 return 0; 1754 } 1755 1756 void * 1757 vmxnet3_dma_allocmem(struct vmxnet3_softc *sc, u_int size, u_int align, bus_addr_t *pa) 1758 { 1759 bus_dma_tag_t t = sc->sc_dmat; 1760 bus_dma_segment_t segs[1]; 1761 bus_dmamap_t map; 1762 caddr_t va; 1763 int n; 1764 1765 if (bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT)) 1766 return NULL; 1767 if (bus_dmamem_map(t, segs, 1, size, &va, BUS_DMA_NOWAIT)) 1768 return NULL; 1769 if (bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &map)) 1770 return NULL; 1771 if (bus_dmamap_load(t, map, va, size, NULL, BUS_DMA_NOWAIT)) 1772 return NULL; 1773 bzero(va, size); 1774 *pa = DMAADDR(map); 1775 bus_dmamap_unload(t, map); 1776 bus_dmamap_destroy(t, map); 1777 return va; 1778 } 1779 1780 static int 1781 vmx_dmamem_alloc(struct vmxnet3_softc *sc, struct vmx_dmamem *vdm, 1782 bus_size_t size, u_int align) 1783 { 1784 vdm->vdm_size = size; 1785 1786 if (bus_dmamap_create(sc->sc_dmat, vdm->vdm_size, 1, 1787 vdm->vdm_size, 0, 1788 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 1789 &vdm->vdm_map) != 0) 1790 return (1); 1791 if (bus_dmamem_alloc(sc->sc_dmat, vdm->vdm_size, 1792 align, 0, &vdm->vdm_seg, 1, &vdm->vdm_nsegs, 1793 BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 1794 goto destroy; 1795 if (bus_dmamem_map(sc->sc_dmat, &vdm->vdm_seg, vdm->vdm_nsegs, 1796 vdm->vdm_size, &vdm->vdm_kva, BUS_DMA_WAITOK) != 0) 1797 goto free; 1798 if (bus_dmamap_load(sc->sc_dmat, vdm->vdm_map, vdm->vdm_kva, 1799 vdm->vdm_size, NULL, BUS_DMA_WAITOK) != 0) 1800 goto unmap; 1801 1802 return (0); 1803 unmap: 1804 bus_dmamem_unmap(sc->sc_dmat, vdm->vdm_kva, vdm->vdm_size); 1805 free: 1806 bus_dmamem_free(sc->sc_dmat, &vdm->vdm_seg, 1); 1807 destroy: 1808 bus_dmamap_destroy(sc->sc_dmat, vdm->vdm_map); 1809 return (1); 1810 } 1811 1812 #ifdef notyet 1813 static void 1814 vmx_dmamem_free(struct vmxnet3_softc *sc, struct vmx_dmamem *vdm) 1815 { 1816 bus_dmamap_unload(sc->sc_dmat, vdm->vdm_map); 1817 bus_dmamem_unmap(sc->sc_dmat, vdm->vdm_kva, vdm->vdm_size); 1818 bus_dmamem_free(sc->sc_dmat, &vdm->vdm_seg, 1); 1819 bus_dmamap_destroy(sc->sc_dmat, vdm->vdm_map); 1820 } 1821 #endif 1822 1823 #if NKSTAT > 0 1824 /* 1825 * "hardware" counters are exported as separate kstats for each tx 1826 * and rx ring, but the request for the hypervisor to update the 1827 * stats is done once at the controller level. we limit the number 1828 * of updates at the controller level to a rate of one per second to 1829 * debounce this a bit. 1830 */ 1831 static const struct timeval vmx_kstat_rate = { 1, 0 }; 1832 1833 /* 1834 * all the vmx stats are 64 bit counters, we just need their name and units. 1835 */ 1836 struct vmx_kstat_tpl { 1837 const char *name; 1838 enum kstat_kv_unit unit; 1839 }; 1840 1841 static const struct vmx_kstat_tpl vmx_rx_kstat_tpl[UPT1_RxStats_count] = { 1842 { "LRO packets", KSTAT_KV_U_PACKETS }, 1843 { "LRO bytes", KSTAT_KV_U_BYTES }, 1844 { "ucast packets", KSTAT_KV_U_PACKETS }, 1845 { "ucast bytes", KSTAT_KV_U_BYTES }, 1846 { "mcast packets", KSTAT_KV_U_PACKETS }, 1847 { "mcast bytes", KSTAT_KV_U_BYTES }, 1848 { "bcast packets", KSTAT_KV_U_PACKETS }, 1849 { "bcast bytes", KSTAT_KV_U_BYTES }, 1850 { "no buffers", KSTAT_KV_U_PACKETS }, 1851 { "errors", KSTAT_KV_U_PACKETS }, 1852 }; 1853 1854 static const struct vmx_kstat_tpl vmx_tx_kstat_tpl[UPT1_TxStats_count] = { 1855 { "TSO packets", KSTAT_KV_U_PACKETS }, 1856 { "TSO bytes", KSTAT_KV_U_BYTES }, 1857 { "ucast packets", KSTAT_KV_U_PACKETS }, 1858 { "ucast bytes", KSTAT_KV_U_BYTES }, 1859 { "mcast packets", KSTAT_KV_U_PACKETS }, 1860 { "mcast bytes", KSTAT_KV_U_BYTES }, 1861 { "bcast packets", KSTAT_KV_U_PACKETS }, 1862 { "bcast bytes", KSTAT_KV_U_BYTES }, 1863 { "errors", KSTAT_KV_U_PACKETS }, 1864 { "discards", KSTAT_KV_U_PACKETS }, 1865 }; 1866 1867 static void 1868 vmx_kstat_init(struct vmxnet3_softc *sc) 1869 { 1870 rw_init(&sc->sc_kstat_lock, "vmxkstat"); 1871 } 1872 1873 static int 1874 vmx_kstat_read(struct kstat *ks) 1875 { 1876 struct vmxnet3_softc *sc = ks->ks_softc; 1877 struct kstat_kv *kvs = ks->ks_data; 1878 uint64_t *vs = ks->ks_ptr; 1879 unsigned int n, i; 1880 1881 if (ratecheck(&sc->sc_kstat_updated, &vmx_kstat_rate)) { 1882 WRITE_CMD(sc, VMXNET3_CMD_GET_STATS); 1883 /* barrier? */ 1884 } 1885 1886 n = ks->ks_datalen / sizeof(*kvs); 1887 for (i = 0; i < n; i++) 1888 kstat_kv_u64(&kvs[i]) = lemtoh64(&vs[i]); 1889 1890 TIMEVAL_TO_TIMESPEC(&sc->sc_kstat_updated, &ks->ks_updated); 1891 1892 return (0); 1893 } 1894 1895 static struct kstat * 1896 vmx_kstat_create(struct vmxnet3_softc *sc, const char *name, unsigned int unit, 1897 const struct vmx_kstat_tpl *tpls, unsigned int n, uint64_t *vs) 1898 { 1899 struct kstat *ks; 1900 struct kstat_kv *kvs; 1901 unsigned int i; 1902 1903 ks = kstat_create(sc->sc_dev.dv_xname, 0, name, unit, 1904 KSTAT_T_KV, 0); 1905 if (ks == NULL) 1906 return (NULL); 1907 1908 kvs = mallocarray(n, sizeof(*kvs), M_DEVBUF, M_WAITOK|M_ZERO); 1909 for (i = 0; i < n; i++) { 1910 const struct vmx_kstat_tpl *tpl = &tpls[i]; 1911 1912 kstat_kv_unit_init(&kvs[i], tpl->name, 1913 KSTAT_KV_T_COUNTER64, tpl->unit); 1914 } 1915 1916 ks->ks_softc = sc; 1917 kstat_set_wlock(ks, &sc->sc_kstat_lock); 1918 ks->ks_ptr = vs; 1919 ks->ks_data = kvs; 1920 ks->ks_datalen = n * sizeof(*kvs); 1921 ks->ks_read = vmx_kstat_read; 1922 TIMEVAL_TO_TIMESPEC(&vmx_kstat_rate, &ks->ks_interval); 1923 1924 kstat_install(ks); 1925 1926 return (ks); 1927 } 1928 1929 static void 1930 vmx_kstat_txstats(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq, 1931 int unit) 1932 { 1933 tq->txkstat = vmx_kstat_create(sc, "vmx-txstats", unit, 1934 vmx_tx_kstat_tpl, nitems(vmx_tx_kstat_tpl), tq->ts->stats); 1935 } 1936 1937 static void 1938 vmx_kstat_rxstats(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq, 1939 int unit) 1940 { 1941 rq->rxkstat = vmx_kstat_create(sc, "vmx-rxstats", unit, 1942 vmx_rx_kstat_tpl, nitems(vmx_rx_kstat_tpl), rq->rs->stats); 1943 } 1944 #endif /* NKSTAT > 0 */ 1945