1 /*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 /* Driver for VirtIO network devices. */ 28 29 #include <sys/cdefs.h> 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/sockio.h> 35 #include <sys/mbuf.h> 36 #include <sys/malloc.h> 37 #include <sys/module.h> 38 #include <sys/socket.h> 39 #include <sys/sysctl.h> 40 #include <sys/taskqueue.h> 41 #include <sys/random.h> 42 #include <sys/sglist.h> 43 #include <sys/serialize.h> 44 #include <sys/bus.h> 45 #include <sys/rman.h> 46 47 #include <net/ethernet.h> 48 #include <net/if.h> 49 #include <net/if_arp.h> 50 #include <net/if_dl.h> 51 #include <net/if_types.h> 52 #include <net/if_media.h> 53 #include <net/vlan/if_vlan_var.h> 54 #include <net/vlan/if_vlan_ether.h> 55 #include <net/ifq_var.h> 56 57 #include <net/bpf.h> 58 59 #include <netinet/in_systm.h> 60 #include <netinet/in.h> 61 #include <netinet/ip.h> 62 #include <netinet/ip6.h> 63 #include <netinet/udp.h> 64 #include <netinet/tcp.h> 65 #include <netinet/sctp.h> 66 67 #include <dev/virtual/virtio/virtio/virtio.h> 68 #include <dev/virtual/virtio/virtio/virtqueue.h> 69 70 #include "virtio_net.h" 71 #include "virtio_if.h" 72 73 struct vtnet_statistics { 74 unsigned long mbuf_alloc_failed; 75 76 unsigned long rx_frame_too_large; 77 unsigned long rx_enq_replacement_failed; 78 unsigned long rx_mergeable_failed; 79 unsigned long rx_csum_bad_ethtype; 80 unsigned long rx_csum_bad_start; 81 unsigned long rx_csum_bad_ipproto; 82 unsigned long rx_csum_bad_offset; 83 unsigned long rx_csum_failed; 84 unsigned long rx_csum_offloaded; 85 unsigned long rx_task_rescheduled; 86 87 unsigned long tx_csum_offloaded; 88 unsigned long tx_tso_offloaded; 89 unsigned long tx_csum_bad_ethtype; 90 unsigned long tx_tso_bad_ethtype; 91 unsigned long tx_task_rescheduled; 92 }; 93 94 struct vtnet_softc { 95 device_t vtnet_dev; 96 struct ifnet *vtnet_ifp; 97 struct lwkt_serialize vtnet_slz; 98 99 uint32_t vtnet_flags; 100 #define VTNET_FLAG_LINK 0x0001 101 #define VTNET_FLAG_SUSPENDED 0x0002 102 #define VTNET_FLAG_CTRL_VQ 0x0004 103 #define VTNET_FLAG_CTRL_RX 0x0008 104 #define VTNET_FLAG_VLAN_FILTER 0x0010 105 #define VTNET_FLAG_TSO_ECN 0x0020 106 #define VTNET_FLAG_MRG_RXBUFS 0x0040 107 #define VTNET_FLAG_LRO_NOMRG 0x0080 108 109 struct virtqueue *vtnet_rx_vq; 110 struct virtqueue *vtnet_tx_vq; 111 struct virtqueue *vtnet_ctrl_vq; 112 113 struct vtnet_tx_header *vtnet_txhdrarea; 114 uint32_t vtnet_txhdridx; 115 struct vtnet_mac_filter *vtnet_macfilter; 116 117 int vtnet_hdr_size; 118 int vtnet_tx_size; 119 int vtnet_rx_size; 120 int vtnet_rx_process_limit; 121 int vtnet_rx_mbuf_size; 122 int vtnet_rx_mbuf_count; 123 int vtnet_if_flags; 124 int vtnet_watchdog_timer; 125 uint64_t vtnet_features; 126 127 struct task vtnet_cfgchg_task; 128 129 struct vtnet_statistics vtnet_stats; 130 131 struct sysctl_ctx_list vtnet_sysctl_ctx; 132 struct sysctl_oid *vtnet_sysctl_tree; 133 134 struct callout vtnet_tick_ch; 135 136 eventhandler_tag vtnet_vlan_attach; 137 eventhandler_tag vtnet_vlan_detach; 138 139 struct ifmedia vtnet_media; 140 /* 141 * Fake media type; the host does not provide us with 142 * any real media information. 143 */ 144 #define VTNET_MEDIATYPE (IFM_ETHER | IFM_1000_T | IFM_FDX) 145 char vtnet_hwaddr[ETHER_ADDR_LEN]; 146 147 /* 148 * During reset, the host's VLAN filtering table is lost. The 149 * array below is used to restore all the VLANs configured on 150 * this interface after a reset. 151 */ 152 #define VTNET_VLAN_SHADOW_SIZE (4096 / 32) 153 int vtnet_nvlans; 154 uint32_t vtnet_vlan_shadow[VTNET_VLAN_SHADOW_SIZE]; 155 156 char vtnet_mtx_name[16]; 157 }; 158 159 /* 160 * When mergeable buffers are not negotiated, the vtnet_rx_header structure 161 * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to 162 * both keep the VirtIO header and the data non-contiguous and to keep the 163 * frame's payload 4 byte aligned. 164 * 165 * When mergeable buffers are negotiated, the host puts the VirtIO header in 166 * the beginning of the first mbuf's data. 167 */ 168 #define VTNET_RX_HEADER_PAD 4 169 struct vtnet_rx_header { 170 struct virtio_net_hdr vrh_hdr; 171 char vrh_pad[VTNET_RX_HEADER_PAD]; 172 } __packed; 173 174 /* 175 * For each outgoing frame, the vtnet_tx_header below is allocated from 176 * the vtnet_tx_header_zone. 177 */ 178 struct vtnet_tx_header { 179 union { 180 struct virtio_net_hdr hdr; 181 struct virtio_net_hdr_mrg_rxbuf mhdr; 182 } vth_uhdr; 183 184 struct mbuf *vth_mbuf; 185 }; 186 187 MALLOC_DEFINE(M_VTNET, "VTNET_TX", "Outgoing VTNET TX frame header"); 188 189 /* 190 * The VirtIO specification does not place a limit on the number of MAC 191 * addresses the guest driver may request to be filtered. In practice, 192 * the host is constrained by available resources. To simplify this driver, 193 * impose a reasonably high limit of MAC addresses we will filter before 194 * falling back to promiscuous or all-multicast modes. 195 */ 196 #define VTNET_MAX_MAC_ENTRIES 128 197 198 struct vtnet_mac_table { 199 uint32_t nentries; 200 uint8_t macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN]; 201 } __packed; 202 203 struct vtnet_mac_filter { 204 struct vtnet_mac_table vmf_unicast; 205 uint32_t vmf_pad; /* Make tables non-contiguous. */ 206 struct vtnet_mac_table vmf_multicast; 207 }; 208 209 #define VTNET_WATCHDOG_TIMEOUT 5 210 #define VTNET_CSUM_OFFLOAD (CSUM_TCP | CSUM_UDP)// | CSUM_SCTP) 211 212 /* Features desired/implemented by this driver. */ 213 #define VTNET_FEATURES \ 214 (VIRTIO_NET_F_MAC | \ 215 VIRTIO_NET_F_STATUS | \ 216 VIRTIO_NET_F_CTRL_VQ | \ 217 VIRTIO_NET_F_CTRL_RX | \ 218 VIRTIO_NET_F_CTRL_VLAN | \ 219 VIRTIO_NET_F_CSUM | \ 220 VIRTIO_NET_F_HOST_TSO4 | \ 221 VIRTIO_NET_F_HOST_TSO6 | \ 222 VIRTIO_NET_F_HOST_ECN | \ 223 VIRTIO_NET_F_GUEST_CSUM | \ 224 VIRTIO_NET_F_GUEST_TSO4 | \ 225 VIRTIO_NET_F_GUEST_TSO6 | \ 226 VIRTIO_NET_F_GUEST_ECN | \ 227 VIRTIO_NET_F_MRG_RXBUF) 228 229 /* 230 * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us 231 * frames larger than 1514 bytes. We do not yet support software LRO 232 * via tcp_lro_rx(). 233 */ 234 #define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \ 235 VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN) 236 237 #define VTNET_MAX_MTU 65536 238 #define VTNET_MAX_RX_SIZE 65550 239 240 /* 241 * Used to preallocate the Vq indirect descriptors. The first segment 242 * is reserved for the header. 243 */ 244 #define VTNET_MIN_RX_SEGS 2 245 #define VTNET_MAX_RX_SEGS 34 246 #define VTNET_MAX_TX_SEGS 34 247 248 #ifndef CSUM_TSO 249 #define CSUM_TSO 0 250 #endif 251 252 #define IFCAP_TSO4 0x00100 /* can do TCP Segmentation Offload */ 253 #define IFCAP_TSO6 0x00200 /* can do TCP6 Segmentation Offload */ 254 #define IFCAP_LRO 0x00400 /* can do Large Receive Offload */ 255 #define IFCAP_VLAN_HWFILTER 0x10000 /* interface hw can filter vlan tag */ 256 #define IFCAP_VLAN_HWTSO 0x40000 /* can do IFCAP_TSO on VLANs */ 257 258 259 /* 260 * Assert we can receive and transmit the maximum with regular 261 * size clusters. 262 */ 263 CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE); 264 CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU); 265 266 /* 267 * Determine how many mbufs are in each receive buffer. For LRO without 268 * mergeable descriptors, we must allocate an mbuf chain large enough to 269 * hold both the vtnet_rx_header and the maximum receivable data. 270 */ 271 #define VTNET_NEEDED_RX_MBUFS(_sc) \ 272 ((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 : \ 273 howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE, \ 274 (_sc)->vtnet_rx_mbuf_size) 275 276 static int vtnet_modevent(module_t, int, void *); 277 278 static int vtnet_probe(device_t); 279 static int vtnet_attach(device_t); 280 static int vtnet_detach(device_t); 281 static int vtnet_suspend(device_t); 282 static int vtnet_resume(device_t); 283 static int vtnet_shutdown(device_t); 284 static int vtnet_config_change(device_t); 285 286 static void vtnet_negotiate_features(struct vtnet_softc *); 287 static int vtnet_alloc_virtqueues(struct vtnet_softc *); 288 static void vtnet_get_hwaddr(struct vtnet_softc *); 289 static void vtnet_set_hwaddr(struct vtnet_softc *); 290 static int vtnet_is_link_up(struct vtnet_softc *); 291 static void vtnet_update_link_status(struct vtnet_softc *); 292 #if 0 293 static void vtnet_watchdog(struct vtnet_softc *); 294 #endif 295 static void vtnet_config_change_task(void *, int); 296 static int vtnet_change_mtu(struct vtnet_softc *, int); 297 static int vtnet_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 298 299 static int vtnet_init_rx_vq(struct vtnet_softc *); 300 static void vtnet_free_rx_mbufs(struct vtnet_softc *); 301 static void vtnet_free_tx_mbufs(struct vtnet_softc *); 302 static void vtnet_free_ctrl_vq(struct vtnet_softc *); 303 304 static struct mbuf * vtnet_alloc_rxbuf(struct vtnet_softc *, int, 305 struct mbuf **); 306 static int vtnet_replace_rxbuf(struct vtnet_softc *, 307 struct mbuf *, int); 308 static int vtnet_newbuf(struct vtnet_softc *); 309 static void vtnet_discard_merged_rxbuf(struct vtnet_softc *, int); 310 static void vtnet_discard_rxbuf(struct vtnet_softc *, struct mbuf *); 311 static int vtnet_enqueue_rxbuf(struct vtnet_softc *, struct mbuf *); 312 static void vtnet_vlan_tag_remove(struct mbuf *); 313 static int vtnet_rx_csum(struct vtnet_softc *, struct mbuf *, 314 struct virtio_net_hdr *); 315 static int vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int); 316 static int vtnet_rxeof(struct vtnet_softc *, int, int *); 317 static void vtnet_rx_intr_task(void *); 318 static int vtnet_rx_vq_intr(void *); 319 320 static void vtnet_txeof(struct vtnet_softc *); 321 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *, 322 struct virtio_net_hdr *); 323 static int vtnet_enqueue_txbuf(struct vtnet_softc *, struct mbuf **, 324 struct vtnet_tx_header *); 325 static int vtnet_encap(struct vtnet_softc *, struct mbuf **); 326 static void vtnet_start_locked(struct ifnet *, struct ifaltq_subque *); 327 static void vtnet_start(struct ifnet *, struct ifaltq_subque *); 328 static void vtnet_tick(void *); 329 static void vtnet_tx_intr_task(void *); 330 static int vtnet_tx_vq_intr(void *); 331 332 static void vtnet_stop(struct vtnet_softc *); 333 static int vtnet_reinit(struct vtnet_softc *); 334 static void vtnet_init_locked(struct vtnet_softc *); 335 static void vtnet_init(void *); 336 337 static void vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *, 338 struct sglist *, int, int); 339 340 static void vtnet_rx_filter(struct vtnet_softc *sc); 341 static int vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int); 342 static int vtnet_set_promisc(struct vtnet_softc *, int); 343 static int vtnet_set_allmulti(struct vtnet_softc *, int); 344 static void vtnet_rx_filter_mac(struct vtnet_softc *); 345 346 static int vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t); 347 static void vtnet_rx_filter_vlan(struct vtnet_softc *); 348 static void vtnet_set_vlan_filter(struct vtnet_softc *, int, uint16_t); 349 static void vtnet_register_vlan(void *, struct ifnet *, uint16_t); 350 static void vtnet_unregister_vlan(void *, struct ifnet *, uint16_t); 351 352 static int vtnet_ifmedia_upd(struct ifnet *); 353 static void vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *); 354 355 static void vtnet_add_statistics(struct vtnet_softc *); 356 357 static int vtnet_enable_rx_intr(struct vtnet_softc *); 358 static int vtnet_enable_tx_intr(struct vtnet_softc *); 359 static void vtnet_disable_rx_intr(struct vtnet_softc *); 360 static void vtnet_disable_tx_intr(struct vtnet_softc *); 361 362 /* Tunables. */ 363 static int vtnet_csum_disable = 0; 364 TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable); 365 static int vtnet_tso_disable = 1; 366 TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable); 367 static int vtnet_lro_disable = 1; 368 TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable); 369 370 /* 371 * Reducing the number of transmit completed interrupts can 372 * improve performance. To do so, the define below keeps the 373 * Tx vq interrupt disabled and adds calls to vtnet_txeof() 374 * in the start and watchdog paths. The price to pay for this 375 * is the m_free'ing of transmitted mbufs may be delayed until 376 * the watchdog fires. 377 */ 378 #define VTNET_TX_INTR_MODERATION 379 380 static struct virtio_feature_desc vtnet_feature_desc[] = { 381 { VIRTIO_NET_F_CSUM, "TxChecksum" }, 382 { VIRTIO_NET_F_GUEST_CSUM, "RxChecksum" }, 383 { VIRTIO_NET_F_MAC, "MacAddress" }, 384 { VIRTIO_NET_F_GSO, "TxAllGSO" }, 385 { VIRTIO_NET_F_GUEST_TSO4, "RxTSOv4" }, 386 { VIRTIO_NET_F_GUEST_TSO6, "RxTSOv6" }, 387 { VIRTIO_NET_F_GUEST_ECN, "RxECN" }, 388 { VIRTIO_NET_F_GUEST_UFO, "RxUFO" }, 389 { VIRTIO_NET_F_HOST_TSO4, "TxTSOv4" }, 390 { VIRTIO_NET_F_HOST_TSO6, "TxTSOv6" }, 391 { VIRTIO_NET_F_HOST_ECN, "TxTSOECN" }, 392 { VIRTIO_NET_F_HOST_UFO, "TxUFO" }, 393 { VIRTIO_NET_F_MRG_RXBUF, "MrgRxBuf" }, 394 { VIRTIO_NET_F_STATUS, "Status" }, 395 { VIRTIO_NET_F_CTRL_VQ, "ControlVq" }, 396 { VIRTIO_NET_F_CTRL_RX, "RxMode" }, 397 { VIRTIO_NET_F_CTRL_VLAN, "VLanFilter" }, 398 { VIRTIO_NET_F_CTRL_RX_EXTRA, "RxModeExtra" }, 399 { VIRTIO_NET_F_MQ, "RFS" }, 400 { 0, NULL } 401 }; 402 403 static device_method_t vtnet_methods[] = { 404 /* Device methods. */ 405 DEVMETHOD(device_probe, vtnet_probe), 406 DEVMETHOD(device_attach, vtnet_attach), 407 DEVMETHOD(device_detach, vtnet_detach), 408 DEVMETHOD(device_suspend, vtnet_suspend), 409 DEVMETHOD(device_resume, vtnet_resume), 410 DEVMETHOD(device_shutdown, vtnet_shutdown), 411 412 /* VirtIO methods. */ 413 DEVMETHOD(virtio_config_change, vtnet_config_change), 414 415 { 0, 0 } 416 }; 417 418 static driver_t vtnet_driver = { 419 "vtnet", 420 vtnet_methods, 421 sizeof(struct vtnet_softc) 422 }; 423 424 static devclass_t vtnet_devclass; 425 426 DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass, 427 vtnet_modevent, 0); 428 MODULE_VERSION(vtnet, 1); 429 MODULE_DEPEND(vtnet, virtio, 1, 1, 1); 430 431 static int 432 vtnet_modevent(module_t mod, int type, void *unused) 433 { 434 int error; 435 436 error = 0; 437 438 switch (type) { 439 case MOD_LOAD: 440 break; 441 case MOD_UNLOAD: 442 break; 443 case MOD_SHUTDOWN: 444 break; 445 default: 446 error = EOPNOTSUPP; 447 break; 448 } 449 450 return (error); 451 } 452 453 static int 454 vtnet_probe(device_t dev) 455 { 456 if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK) 457 return (ENXIO); 458 459 device_set_desc(dev, "VirtIO Networking Adapter"); 460 461 return (BUS_PROBE_DEFAULT); 462 } 463 464 static int 465 vtnet_attach(device_t dev) 466 { 467 struct vtnet_softc *sc; 468 struct ifnet *ifp; 469 int tx_size, error; 470 471 sc = device_get_softc(dev); 472 sc->vtnet_dev = dev; 473 474 lwkt_serialize_init(&sc->vtnet_slz); 475 callout_init(&sc->vtnet_tick_ch); 476 477 ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd, 478 vtnet_ifmedia_sts); 479 ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL); 480 ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE); 481 482 vtnet_add_statistics(sc); 483 484 virtio_set_feature_desc(dev, vtnet_feature_desc); 485 vtnet_negotiate_features(sc); 486 487 if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) { 488 sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS; 489 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 490 } else { 491 sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 492 } 493 494 sc->vtnet_rx_mbuf_size = MCLBYTES; 495 sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc); 496 497 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) { 498 sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ; 499 500 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_RX)) 501 sc->vtnet_flags |= VTNET_FLAG_CTRL_RX; 502 if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VLAN)) 503 sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER; 504 } 505 506 vtnet_get_hwaddr(sc); 507 508 error = vtnet_alloc_virtqueues(sc); 509 if (error) { 510 device_printf(dev, "cannot allocate virtqueues\n"); 511 goto fail; 512 } 513 514 ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER); 515 if (ifp == NULL) { 516 device_printf(dev, "cannot allocate ifnet structure\n"); 517 error = ENOSPC; 518 goto fail; 519 } 520 521 ifp->if_softc = sc; 522 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 523 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 524 ifp->if_init = vtnet_init; 525 ifp->if_start = vtnet_start; 526 ifp->if_ioctl = vtnet_ioctl; 527 528 sc->vtnet_rx_size = virtqueue_size(sc->vtnet_rx_vq); 529 sc->vtnet_rx_process_limit = sc->vtnet_rx_size; 530 531 tx_size = virtqueue_size(sc->vtnet_tx_vq); 532 sc->vtnet_tx_size = tx_size; 533 sc->vtnet_txhdridx = 0; 534 sc->vtnet_txhdrarea = contigmalloc( 535 ((sc->vtnet_tx_size / 2) + 1) * sizeof(struct vtnet_tx_header), 536 M_VTNET, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0); 537 if (sc->vtnet_txhdrarea == NULL) { 538 device_printf(dev, "cannot contigmalloc the tx headers\n"); 539 goto fail; 540 } 541 sc->vtnet_macfilter = contigmalloc( 542 sizeof(struct vtnet_mac_filter), 543 M_DEVBUF, M_WAITOK, 0, BUS_SPACE_MAXADDR, 4, 0); 544 if (sc->vtnet_macfilter == NULL) { 545 device_printf(dev, 546 "cannot contigmalloc the mac filter table\n"); 547 goto fail; 548 } 549 ifq_set_maxlen(&ifp->if_snd, tx_size - 1); 550 ifq_set_ready(&ifp->if_snd); 551 552 ether_ifattach(ifp, sc->vtnet_hwaddr, NULL); 553 554 if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS)){ 555 //ifp->if_capabilities |= IFCAP_LINKSTATE; 556 kprintf("add dynamic link state\n"); 557 } 558 559 /* Tell the upper layer(s) we support long frames. */ 560 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 561 ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU; 562 563 if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) { 564 ifp->if_capabilities |= IFCAP_TXCSUM; 565 566 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4)) 567 ifp->if_capabilities |= IFCAP_TSO4; 568 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6)) 569 ifp->if_capabilities |= IFCAP_TSO6; 570 if (ifp->if_capabilities & IFCAP_TSO) 571 ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 572 573 if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN)) 574 sc->vtnet_flags |= VTNET_FLAG_TSO_ECN; 575 } 576 577 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) { 578 ifp->if_capabilities |= IFCAP_RXCSUM; 579 580 if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) || 581 virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6)) 582 ifp->if_capabilities |= IFCAP_LRO; 583 } 584 585 if (ifp->if_capabilities & IFCAP_HWCSUM) { 586 /* 587 * VirtIO does not support VLAN tagging, but we can fake 588 * it by inserting and removing the 802.1Q header during 589 * transmit and receive. We are then able to do checksum 590 * offloading of VLAN frames. 591 */ 592 ifp->if_capabilities |= 593 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; 594 } 595 596 ifp->if_capenable = ifp->if_capabilities; 597 598 /* 599 * Capabilities after here are not enabled by default. 600 */ 601 602 if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) { 603 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 604 605 sc->vtnet_vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 606 vtnet_register_vlan, sc, EVENTHANDLER_PRI_FIRST); 607 sc->vtnet_vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 608 vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST); 609 } 610 611 TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc); 612 613 error = virtio_setup_intr(dev, &sc->vtnet_slz); 614 if (error) { 615 device_printf(dev, "cannot setup virtqueue interrupts\n"); 616 ether_ifdetach(ifp); 617 goto fail; 618 } 619 620 /* 621 * Device defaults to promiscuous mode for backwards 622 * compatibility. Turn it off if possible. 623 */ 624 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 625 lwkt_serialize_enter(&sc->vtnet_slz); 626 if (vtnet_set_promisc(sc, 0) != 0) { 627 ifp->if_flags |= IFF_PROMISC; 628 device_printf(dev, 629 "cannot disable promiscuous mode\n"); 630 } 631 lwkt_serialize_exit(&sc->vtnet_slz); 632 } else 633 ifp->if_flags |= IFF_PROMISC; 634 635 fail: 636 if (error) 637 vtnet_detach(dev); 638 639 return (error); 640 } 641 642 static int 643 vtnet_detach(device_t dev) 644 { 645 struct vtnet_softc *sc; 646 struct ifnet *ifp; 647 648 sc = device_get_softc(dev); 649 ifp = sc->vtnet_ifp; 650 651 if (device_is_attached(dev)) { 652 lwkt_serialize_enter(&sc->vtnet_slz); 653 vtnet_stop(sc); 654 lwkt_serialize_exit(&sc->vtnet_slz); 655 656 callout_stop(&sc->vtnet_tick_ch); 657 taskqueue_drain(taskqueue_swi, &sc->vtnet_cfgchg_task); 658 659 ether_ifdetach(ifp); 660 } 661 662 if (sc->vtnet_vlan_attach != NULL) { 663 EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach); 664 sc->vtnet_vlan_attach = NULL; 665 } 666 if (sc->vtnet_vlan_detach != NULL) { 667 EVENTHANDLER_DEREGISTER(vlan_unconfg, sc->vtnet_vlan_detach); 668 sc->vtnet_vlan_detach = NULL; 669 } 670 671 if (ifp) { 672 if_free(ifp); 673 sc->vtnet_ifp = NULL; 674 } 675 676 if (sc->vtnet_rx_vq != NULL) 677 vtnet_free_rx_mbufs(sc); 678 if (sc->vtnet_tx_vq != NULL) 679 vtnet_free_tx_mbufs(sc); 680 if (sc->vtnet_ctrl_vq != NULL) 681 vtnet_free_ctrl_vq(sc); 682 683 if (sc->vtnet_txhdrarea != NULL) { 684 contigfree(sc->vtnet_txhdrarea, 685 ((sc->vtnet_tx_size / 2) + 1) * 686 sizeof(struct vtnet_tx_header), M_VTNET); 687 sc->vtnet_txhdrarea = NULL; 688 } 689 if (sc->vtnet_macfilter != NULL) { 690 contigfree(sc->vtnet_macfilter, 691 sizeof(struct vtnet_mac_filter), M_DEVBUF); 692 sc->vtnet_macfilter = NULL; 693 } 694 695 ifmedia_removeall(&sc->vtnet_media); 696 697 return (0); 698 } 699 700 static int 701 vtnet_suspend(device_t dev) 702 { 703 struct vtnet_softc *sc; 704 705 sc = device_get_softc(dev); 706 707 lwkt_serialize_enter(&sc->vtnet_slz); 708 vtnet_stop(sc); 709 sc->vtnet_flags |= VTNET_FLAG_SUSPENDED; 710 lwkt_serialize_exit(&sc->vtnet_slz); 711 712 return (0); 713 } 714 715 static int 716 vtnet_resume(device_t dev) 717 { 718 struct vtnet_softc *sc; 719 struct ifnet *ifp; 720 721 sc = device_get_softc(dev); 722 ifp = sc->vtnet_ifp; 723 724 lwkt_serialize_enter(&sc->vtnet_slz); 725 if (ifp->if_flags & IFF_UP) 726 vtnet_init_locked(sc); 727 sc->vtnet_flags &= ~VTNET_FLAG_SUSPENDED; 728 lwkt_serialize_exit(&sc->vtnet_slz); 729 730 return (0); 731 } 732 733 static int 734 vtnet_shutdown(device_t dev) 735 { 736 737 /* 738 * Suspend already does all of what we need to 739 * do here; we just never expect to be resumed. 740 */ 741 return (vtnet_suspend(dev)); 742 } 743 744 static int 745 vtnet_config_change(device_t dev) 746 { 747 struct vtnet_softc *sc; 748 749 sc = device_get_softc(dev); 750 751 taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtnet_cfgchg_task); 752 753 return (1); 754 } 755 756 static void 757 vtnet_negotiate_features(struct vtnet_softc *sc) 758 { 759 device_t dev; 760 uint64_t mask, features; 761 762 dev = sc->vtnet_dev; 763 mask = 0; 764 765 if (vtnet_csum_disable) 766 mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM; 767 768 /* 769 * TSO and LRO are only available when their corresponding 770 * checksum offload feature is also negotiated. 771 */ 772 773 if (vtnet_csum_disable || vtnet_tso_disable) 774 mask |= VIRTIO_NET_F_HOST_TSO4 | VIRTIO_NET_F_HOST_TSO6 | 775 VIRTIO_NET_F_HOST_ECN; 776 777 if (vtnet_csum_disable || vtnet_lro_disable) 778 mask |= VTNET_LRO_FEATURES; 779 780 features = VTNET_FEATURES & ~mask; 781 features |= VIRTIO_F_NOTIFY_ON_EMPTY; 782 sc->vtnet_features = virtio_negotiate_features(dev, features); 783 } 784 785 static int 786 vtnet_alloc_virtqueues(struct vtnet_softc *sc) 787 { 788 device_t dev; 789 struct vq_alloc_info vq_info[3]; 790 int nvqs, rxsegs; 791 792 dev = sc->vtnet_dev; 793 nvqs = 2; 794 795 /* 796 * Indirect descriptors are not needed for the Rx 797 * virtqueue when mergeable buffers are negotiated. 798 * The header is placed inline with the data, not 799 * in a separate descriptor, and mbuf clusters are 800 * always physically contiguous. 801 */ 802 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 803 rxsegs = sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG ? 804 VTNET_MAX_RX_SEGS : VTNET_MIN_RX_SEGS; 805 } else 806 rxsegs = 0; 807 808 VQ_ALLOC_INFO_INIT(&vq_info[0], rxsegs, 809 vtnet_rx_vq_intr, sc, &sc->vtnet_rx_vq, 810 "%s receive", device_get_nameunit(dev)); 811 812 VQ_ALLOC_INFO_INIT(&vq_info[1], VTNET_MAX_TX_SEGS, 813 vtnet_tx_vq_intr, sc, &sc->vtnet_tx_vq, 814 "%s transmit", device_get_nameunit(dev)); 815 816 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 817 nvqs++; 818 819 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, NULL, NULL, 820 &sc->vtnet_ctrl_vq, "%s control", 821 device_get_nameunit(dev)); 822 } 823 824 return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info)); 825 } 826 827 static void 828 vtnet_get_hwaddr(struct vtnet_softc *sc) 829 { 830 device_t dev; 831 832 dev = sc->vtnet_dev; 833 834 if (virtio_with_feature(dev, VIRTIO_NET_F_MAC)) { 835 virtio_read_device_config(dev, 836 offsetof(struct virtio_net_config, mac), 837 sc->vtnet_hwaddr, ETHER_ADDR_LEN); 838 } else { 839 /* Generate random locally administered unicast address. */ 840 sc->vtnet_hwaddr[0] = 0xB2; 841 karc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1); 842 843 vtnet_set_hwaddr(sc); 844 } 845 } 846 847 static void 848 vtnet_set_hwaddr(struct vtnet_softc *sc) 849 { 850 device_t dev; 851 852 dev = sc->vtnet_dev; 853 854 virtio_write_device_config(dev, 855 offsetof(struct virtio_net_config, mac), 856 sc->vtnet_hwaddr, ETHER_ADDR_LEN); 857 } 858 859 static int 860 vtnet_is_link_up(struct vtnet_softc *sc) 861 { 862 device_t dev; 863 struct ifnet *ifp; 864 uint16_t status; 865 866 dev = sc->vtnet_dev; 867 ifp = sc->vtnet_ifp; 868 869 ASSERT_SERIALIZED(&sc->vtnet_slz); 870 871 status = virtio_read_dev_config_2(dev, 872 offsetof(struct virtio_net_config, status)); 873 874 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 875 } 876 877 static void 878 vtnet_update_link_status(struct vtnet_softc *sc) 879 { 880 device_t dev; 881 struct ifnet *ifp; 882 struct ifaltq_subque *ifsq; 883 int link; 884 885 dev = sc->vtnet_dev; 886 ifp = sc->vtnet_ifp; 887 ifsq = ifq_get_subq_default(&ifp->if_snd); 888 889 link = vtnet_is_link_up(sc); 890 891 if (link && ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) { 892 sc->vtnet_flags |= VTNET_FLAG_LINK; 893 if (bootverbose) 894 device_printf(dev, "Link is up\n"); 895 ifp->if_link_state = LINK_STATE_UP; 896 if_link_state_change(ifp); 897 if (!ifsq_is_empty(ifsq)) 898 vtnet_start_locked(ifp, ifsq); 899 } else if (!link && (sc->vtnet_flags & VTNET_FLAG_LINK)) { 900 sc->vtnet_flags &= ~VTNET_FLAG_LINK; 901 if (bootverbose) 902 device_printf(dev, "Link is down\n"); 903 904 ifp->if_link_state = LINK_STATE_DOWN; 905 if_link_state_change(ifp); 906 } 907 } 908 909 #if 0 910 static void 911 vtnet_watchdog(struct vtnet_softc *sc) 912 { 913 struct ifnet *ifp; 914 915 ifp = sc->vtnet_ifp; 916 917 #ifdef VTNET_TX_INTR_MODERATION 918 vtnet_txeof(sc); 919 #endif 920 921 if (sc->vtnet_watchdog_timer == 0 || --sc->vtnet_watchdog_timer) 922 return; 923 924 if_printf(ifp, "watchdog timeout -- resetting\n"); 925 #ifdef VTNET_DEBUG 926 virtqueue_dump(sc->vtnet_tx_vq); 927 #endif 928 ifp->if_oerrors++; 929 ifp->if_flags &= ~IFF_RUNNING; 930 vtnet_init_locked(sc); 931 } 932 #endif 933 934 static void 935 vtnet_config_change_task(void *arg, int pending) 936 { 937 struct vtnet_softc *sc; 938 939 sc = arg; 940 941 lwkt_serialize_enter(&sc->vtnet_slz); 942 vtnet_update_link_status(sc); 943 lwkt_serialize_exit(&sc->vtnet_slz); 944 } 945 946 static int 947 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data,struct ucred *cr) 948 { 949 struct vtnet_softc *sc; 950 struct ifreq *ifr; 951 int reinit, mask, error; 952 953 sc = ifp->if_softc; 954 ifr = (struct ifreq *) data; 955 reinit = 0; 956 error = 0; 957 958 switch (cmd) { 959 case SIOCSIFMTU: 960 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > VTNET_MAX_MTU) 961 error = EINVAL; 962 else if (ifp->if_mtu != ifr->ifr_mtu) { 963 lwkt_serialize_enter(&sc->vtnet_slz); 964 error = vtnet_change_mtu(sc, ifr->ifr_mtu); 965 lwkt_serialize_exit(&sc->vtnet_slz); 966 } 967 break; 968 969 case SIOCSIFFLAGS: 970 lwkt_serialize_enter(&sc->vtnet_slz); 971 if ((ifp->if_flags & IFF_UP) == 0) { 972 if (ifp->if_flags & IFF_RUNNING) 973 vtnet_stop(sc); 974 } else if (ifp->if_flags & IFF_RUNNING) { 975 if ((ifp->if_flags ^ sc->vtnet_if_flags) & 976 (IFF_PROMISC | IFF_ALLMULTI)) { 977 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) 978 vtnet_rx_filter(sc); 979 else 980 error = ENOTSUP; 981 } 982 } else 983 vtnet_init_locked(sc); 984 985 if (error == 0) 986 sc->vtnet_if_flags = ifp->if_flags; 987 lwkt_serialize_exit(&sc->vtnet_slz); 988 break; 989 990 case SIOCADDMULTI: 991 case SIOCDELMULTI: 992 lwkt_serialize_enter(&sc->vtnet_slz); 993 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) && 994 (ifp->if_flags & IFF_RUNNING)) 995 vtnet_rx_filter_mac(sc); 996 lwkt_serialize_exit(&sc->vtnet_slz); 997 break; 998 999 case SIOCSIFMEDIA: 1000 case SIOCGIFMEDIA: 1001 error = ifmedia_ioctl(ifp, ifr, &sc->vtnet_media, cmd); 1002 break; 1003 1004 case SIOCSIFCAP: 1005 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1006 1007 lwkt_serialize_enter(&sc->vtnet_slz); 1008 1009 if (mask & IFCAP_TXCSUM) { 1010 ifp->if_capenable ^= IFCAP_TXCSUM; 1011 if (ifp->if_capenable & IFCAP_TXCSUM) 1012 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD; 1013 else 1014 ifp->if_hwassist &= ~VTNET_CSUM_OFFLOAD; 1015 } 1016 1017 if (mask & IFCAP_TSO4) { 1018 ifp->if_capenable ^= IFCAP_TSO4; 1019 if (ifp->if_capenable & IFCAP_TSO4) 1020 ifp->if_hwassist |= CSUM_TSO; 1021 else 1022 ifp->if_hwassist &= ~CSUM_TSO; 1023 } 1024 1025 if (mask & IFCAP_RXCSUM) { 1026 ifp->if_capenable ^= IFCAP_RXCSUM; 1027 reinit = 1; 1028 } 1029 1030 if (mask & IFCAP_LRO) { 1031 ifp->if_capenable ^= IFCAP_LRO; 1032 reinit = 1; 1033 } 1034 1035 if (mask & IFCAP_VLAN_HWFILTER) { 1036 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1037 reinit = 1; 1038 } 1039 1040 if (mask & IFCAP_VLAN_HWTSO) 1041 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1042 1043 if (mask & IFCAP_VLAN_HWTAGGING) 1044 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1045 1046 if (reinit && (ifp->if_flags & IFF_RUNNING)) { 1047 ifp->if_flags &= ~IFF_RUNNING; 1048 vtnet_init_locked(sc); 1049 } 1050 //VLAN_CAPABILITIES(ifp); 1051 1052 lwkt_serialize_exit(&sc->vtnet_slz); 1053 break; 1054 1055 default: 1056 error = ether_ioctl(ifp, cmd, data); 1057 break; 1058 } 1059 1060 return (error); 1061 } 1062 1063 static int 1064 vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu) 1065 { 1066 struct ifnet *ifp; 1067 int new_frame_size, clsize; 1068 1069 ifp = sc->vtnet_ifp; 1070 1071 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1072 new_frame_size = sizeof(struct vtnet_rx_header) + 1073 sizeof(struct ether_vlan_header) + new_mtu; 1074 1075 if (new_frame_size > MJUM9BYTES) 1076 return (EINVAL); 1077 1078 if (new_frame_size <= MCLBYTES) 1079 clsize = MCLBYTES; 1080 else 1081 clsize = MJUM9BYTES; 1082 } else { 1083 new_frame_size = sizeof(struct virtio_net_hdr_mrg_rxbuf) + 1084 sizeof(struct ether_vlan_header) + new_mtu; 1085 1086 if (new_frame_size <= MCLBYTES) 1087 clsize = MCLBYTES; 1088 else 1089 clsize = MJUMPAGESIZE; 1090 } 1091 1092 sc->vtnet_rx_mbuf_size = clsize; 1093 sc->vtnet_rx_mbuf_count = VTNET_NEEDED_RX_MBUFS(sc); 1094 KASSERT(sc->vtnet_rx_mbuf_count < VTNET_MAX_RX_SEGS, 1095 ("too many rx mbufs: %d", sc->vtnet_rx_mbuf_count)); 1096 1097 ifp->if_mtu = new_mtu; 1098 1099 if (ifp->if_flags & IFF_RUNNING) { 1100 ifp->if_flags &= ~IFF_RUNNING; 1101 vtnet_init_locked(sc); 1102 } 1103 1104 return (0); 1105 } 1106 1107 static int 1108 vtnet_init_rx_vq(struct vtnet_softc *sc) 1109 { 1110 struct virtqueue *vq; 1111 int nbufs, error; 1112 1113 vq = sc->vtnet_rx_vq; 1114 nbufs = 0; 1115 error = ENOSPC; 1116 1117 while (!virtqueue_full(vq)) { 1118 if ((error = vtnet_newbuf(sc)) != 0) 1119 break; 1120 nbufs++; 1121 } 1122 1123 if (nbufs > 0) { 1124 virtqueue_notify(vq, &sc->vtnet_slz); 1125 1126 /* 1127 * EMSGSIZE signifies the virtqueue did not have enough 1128 * entries available to hold the last mbuf. This is not 1129 * an error. We should not get ENOSPC since we check if 1130 * the virtqueue is full before attempting to add a 1131 * buffer. 1132 */ 1133 if (error == EMSGSIZE) 1134 error = 0; 1135 } 1136 1137 return (error); 1138 } 1139 1140 static void 1141 vtnet_free_rx_mbufs(struct vtnet_softc *sc) 1142 { 1143 struct virtqueue *vq; 1144 struct mbuf *m; 1145 int last; 1146 1147 vq = sc->vtnet_rx_vq; 1148 last = 0; 1149 1150 while ((m = virtqueue_drain(vq, &last)) != NULL) 1151 m_freem(m); 1152 1153 KASSERT(virtqueue_empty(vq), ("mbufs remaining in Rx Vq")); 1154 } 1155 1156 static void 1157 vtnet_free_tx_mbufs(struct vtnet_softc *sc) 1158 { 1159 struct virtqueue *vq; 1160 struct vtnet_tx_header *txhdr; 1161 int last; 1162 1163 vq = sc->vtnet_tx_vq; 1164 last = 0; 1165 1166 while ((txhdr = virtqueue_drain(vq, &last)) != NULL) { 1167 m_freem(txhdr->vth_mbuf); 1168 } 1169 1170 KASSERT(virtqueue_empty(vq), ("mbufs remaining in Tx Vq")); 1171 } 1172 1173 static void 1174 vtnet_free_ctrl_vq(struct vtnet_softc *sc) 1175 { 1176 /* 1177 * The control virtqueue is only polled, therefore 1178 * it should already be empty. 1179 */ 1180 KASSERT(virtqueue_empty(sc->vtnet_ctrl_vq), 1181 ("Ctrl Vq not empty")); 1182 } 1183 1184 static struct mbuf * 1185 vtnet_alloc_rxbuf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp) 1186 { 1187 struct mbuf *m_head, *m_tail, *m; 1188 int i, clsize; 1189 1190 clsize = sc->vtnet_rx_mbuf_size; 1191 1192 /*use getcl instead of getjcl. see if_mxge.c comment line 2398*/ 1193 //m_head = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, clsize); 1194 m_head = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR ); 1195 if (m_head == NULL) 1196 goto fail; 1197 1198 m_head->m_len = clsize; 1199 m_tail = m_head; 1200 1201 if (nbufs > 1) { 1202 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, 1203 ("chained Rx mbuf requested without LRO_NOMRG")); 1204 1205 for (i = 0; i < nbufs - 1; i++) { 1206 //m = m_getjcl(M_DONTWAIT, MT_DATA, 0, clsize); 1207 m = m_getcl(MB_DONTWAIT, MT_DATA, 0); 1208 if (m == NULL) 1209 goto fail; 1210 1211 m->m_len = clsize; 1212 m_tail->m_next = m; 1213 m_tail = m; 1214 } 1215 } 1216 1217 if (m_tailp != NULL) 1218 *m_tailp = m_tail; 1219 1220 return (m_head); 1221 1222 fail: 1223 sc->vtnet_stats.mbuf_alloc_failed++; 1224 m_freem(m_head); 1225 1226 return (NULL); 1227 } 1228 1229 static int 1230 vtnet_replace_rxbuf(struct vtnet_softc *sc, struct mbuf *m0, int len0) 1231 { 1232 struct mbuf *m, *m_prev; 1233 struct mbuf *m_new, *m_tail; 1234 int len, clsize, nreplace, error; 1235 1236 m = m0; 1237 m_prev = NULL; 1238 len = len0; 1239 1240 m_tail = NULL; 1241 clsize = sc->vtnet_rx_mbuf_size; 1242 nreplace = 0; 1243 1244 if (m->m_next != NULL) 1245 KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG, 1246 ("chained Rx mbuf without LRO_NOMRG")); 1247 1248 /* 1249 * Since LRO_NOMRG mbuf chains are so large, we want to avoid 1250 * allocating an entire chain for each received frame. When 1251 * the received frame's length is less than that of the chain, 1252 * the unused mbufs are reassigned to the new chain. 1253 */ 1254 while (len > 0) { 1255 /* 1256 * Something is seriously wrong if we received 1257 * a frame larger than the mbuf chain. Drop it. 1258 */ 1259 if (m == NULL) { 1260 sc->vtnet_stats.rx_frame_too_large++; 1261 return (EMSGSIZE); 1262 } 1263 1264 KASSERT(m->m_len == clsize, 1265 ("mbuf length not expected cluster size: %d", 1266 m->m_len)); 1267 1268 m->m_len = MIN(m->m_len, len); 1269 len -= m->m_len; 1270 1271 m_prev = m; 1272 m = m->m_next; 1273 nreplace++; 1274 } 1275 1276 KASSERT(m_prev != NULL, ("m_prev == NULL")); 1277 KASSERT(nreplace <= sc->vtnet_rx_mbuf_count, 1278 ("too many replacement mbufs: %d/%d", nreplace, 1279 sc->vtnet_rx_mbuf_count)); 1280 1281 m_new = vtnet_alloc_rxbuf(sc, nreplace, &m_tail); 1282 if (m_new == NULL) { 1283 m_prev->m_len = clsize; 1284 return (ENOBUFS); 1285 } 1286 1287 /* 1288 * Move unused mbufs, if any, from the original chain 1289 * onto the end of the new chain. 1290 */ 1291 if (m_prev->m_next != NULL) { 1292 m_tail->m_next = m_prev->m_next; 1293 m_prev->m_next = NULL; 1294 } 1295 1296 error = vtnet_enqueue_rxbuf(sc, m_new); 1297 if (error) { 1298 /* 1299 * BAD! We could not enqueue the replacement mbuf chain. We 1300 * must restore the m0 chain to the original state if it was 1301 * modified so we can subsequently discard it. 1302 * 1303 * NOTE: The replacement is suppose to be an identical copy 1304 * to the one just dequeued so this is an unexpected error. 1305 */ 1306 sc->vtnet_stats.rx_enq_replacement_failed++; 1307 1308 if (m_tail->m_next != NULL) { 1309 m_prev->m_next = m_tail->m_next; 1310 m_tail->m_next = NULL; 1311 } 1312 1313 m_prev->m_len = clsize; 1314 m_freem(m_new); 1315 } 1316 1317 return (error); 1318 } 1319 1320 static int 1321 vtnet_newbuf(struct vtnet_softc *sc) 1322 { 1323 struct mbuf *m; 1324 int error; 1325 1326 m = vtnet_alloc_rxbuf(sc, sc->vtnet_rx_mbuf_count, NULL); 1327 if (m == NULL) 1328 return (ENOBUFS); 1329 1330 error = vtnet_enqueue_rxbuf(sc, m); 1331 if (error) 1332 m_freem(m); 1333 1334 return (error); 1335 } 1336 1337 static void 1338 vtnet_discard_merged_rxbuf(struct vtnet_softc *sc, int nbufs) 1339 { 1340 struct virtqueue *vq; 1341 struct mbuf *m; 1342 1343 vq = sc->vtnet_rx_vq; 1344 1345 while (--nbufs > 0) { 1346 if ((m = virtqueue_dequeue(vq, NULL)) == NULL) 1347 break; 1348 vtnet_discard_rxbuf(sc, m); 1349 } 1350 } 1351 1352 static void 1353 vtnet_discard_rxbuf(struct vtnet_softc *sc, struct mbuf *m) 1354 { 1355 int error; 1356 1357 /* 1358 * Requeue the discarded mbuf. This should always be 1359 * successful since it was just dequeued. 1360 */ 1361 error = vtnet_enqueue_rxbuf(sc, m); 1362 KASSERT(error == 0, ("cannot requeue discarded mbuf")); 1363 } 1364 1365 static int 1366 vtnet_enqueue_rxbuf(struct vtnet_softc *sc, struct mbuf *m) 1367 { 1368 struct sglist sg; 1369 struct sglist_seg segs[VTNET_MAX_RX_SEGS]; 1370 struct vtnet_rx_header *rxhdr; 1371 struct virtio_net_hdr *hdr; 1372 uint8_t *mdata; 1373 int offset, error; 1374 1375 ASSERT_SERIALIZED(&sc->vtnet_slz); 1376 if ((sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0) 1377 KASSERT(m->m_next == NULL, ("chained Rx mbuf")); 1378 1379 sglist_init(&sg, VTNET_MAX_RX_SEGS, segs); 1380 1381 mdata = mtod(m, uint8_t *); 1382 offset = 0; 1383 1384 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1385 rxhdr = (struct vtnet_rx_header *) mdata; 1386 hdr = &rxhdr->vrh_hdr; 1387 offset += sizeof(struct vtnet_rx_header); 1388 1389 error = sglist_append(&sg, hdr, sc->vtnet_hdr_size); 1390 KASSERT(error == 0, ("cannot add header to sglist")); 1391 } 1392 1393 error = sglist_append(&sg, mdata + offset, m->m_len - offset); 1394 if (error) 1395 return (error); 1396 1397 if (m->m_next != NULL) { 1398 error = sglist_append_mbuf(&sg, m->m_next); 1399 if (error) 1400 return (error); 1401 } 1402 1403 return (virtqueue_enqueue(sc->vtnet_rx_vq, m, &sg, 0, sg.sg_nseg)); 1404 } 1405 1406 static void 1407 vtnet_vlan_tag_remove(struct mbuf *m) 1408 { 1409 struct ether_vlan_header *evl; 1410 1411 evl = mtod(m, struct ether_vlan_header *); 1412 1413 m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag); 1414 m->m_flags |= M_VLANTAG; 1415 1416 /* Strip the 802.1Q header. */ 1417 bcopy((char *) evl, (char *) evl + ETHER_VLAN_ENCAP_LEN, 1418 ETHER_HDR_LEN - ETHER_TYPE_LEN); 1419 m_adj(m, ETHER_VLAN_ENCAP_LEN); 1420 } 1421 1422 /* 1423 * Alternative method of doing receive checksum offloading. Rather 1424 * than parsing the received frame down to the IP header, use the 1425 * csum_offset to determine which CSUM_* flags are appropriate. We 1426 * can get by with doing this only because the checksum offsets are 1427 * unique for the things we care about. 1428 */ 1429 static int 1430 vtnet_rx_csum(struct vtnet_softc *sc, struct mbuf *m, 1431 struct virtio_net_hdr *hdr) 1432 { 1433 struct ether_header *eh; 1434 struct ether_vlan_header *evh; 1435 struct udphdr *udp; 1436 int csum_len; 1437 uint16_t eth_type; 1438 1439 csum_len = hdr->csum_start + hdr->csum_offset; 1440 1441 if (csum_len < sizeof(struct ether_header) + sizeof(struct ip)) 1442 return (1); 1443 if (m->m_len < csum_len) 1444 return (1); 1445 1446 eh = mtod(m, struct ether_header *); 1447 eth_type = ntohs(eh->ether_type); 1448 if (eth_type == ETHERTYPE_VLAN) { 1449 evh = mtod(m, struct ether_vlan_header *); 1450 eth_type = ntohs(evh->evl_proto); 1451 } 1452 1453 if (eth_type != ETHERTYPE_IP && eth_type != ETHERTYPE_IPV6) { 1454 sc->vtnet_stats.rx_csum_bad_ethtype++; 1455 return (1); 1456 } 1457 1458 /* Use the offset to determine the appropriate CSUM_* flags. */ 1459 switch (hdr->csum_offset) { 1460 case offsetof(struct udphdr, uh_sum): 1461 if (m->m_len < hdr->csum_start + sizeof(struct udphdr)) 1462 return (1); 1463 udp = (struct udphdr *)(mtod(m, uint8_t *) + hdr->csum_start); 1464 if (udp->uh_sum == 0) 1465 return (0); 1466 1467 /* FALLTHROUGH */ 1468 1469 case offsetof(struct tcphdr, th_sum): 1470 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 1471 m->m_pkthdr.csum_data = 0xFFFF; 1472 break; 1473 1474 case offsetof(struct sctphdr, checksum): 1475 //m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID; 1476 break; 1477 1478 default: 1479 sc->vtnet_stats.rx_csum_bad_offset++; 1480 return (1); 1481 } 1482 1483 sc->vtnet_stats.rx_csum_offloaded++; 1484 1485 return (0); 1486 } 1487 1488 static int 1489 vtnet_rxeof_merged(struct vtnet_softc *sc, struct mbuf *m_head, int nbufs) 1490 { 1491 struct ifnet *ifp; 1492 struct virtqueue *vq; 1493 struct mbuf *m, *m_tail; 1494 int len; 1495 1496 ifp = sc->vtnet_ifp; 1497 vq = sc->vtnet_rx_vq; 1498 m_tail = m_head; 1499 1500 while (--nbufs > 0) { 1501 m = virtqueue_dequeue(vq, &len); 1502 if (m == NULL) { 1503 ifp->if_ierrors++; 1504 goto fail; 1505 } 1506 1507 if (vtnet_newbuf(sc) != 0) { 1508 ifp->if_iqdrops++; 1509 vtnet_discard_rxbuf(sc, m); 1510 if (nbufs > 1) 1511 vtnet_discard_merged_rxbuf(sc, nbufs); 1512 goto fail; 1513 } 1514 1515 if (m->m_len < len) 1516 len = m->m_len; 1517 1518 m->m_len = len; 1519 m->m_flags &= ~M_PKTHDR; 1520 1521 m_head->m_pkthdr.len += len; 1522 m_tail->m_next = m; 1523 m_tail = m; 1524 } 1525 1526 return (0); 1527 1528 fail: 1529 sc->vtnet_stats.rx_mergeable_failed++; 1530 m_freem(m_head); 1531 1532 return (1); 1533 } 1534 1535 static int 1536 vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp) 1537 { 1538 struct virtio_net_hdr lhdr; 1539 struct ifnet *ifp; 1540 struct virtqueue *vq; 1541 struct mbuf *m; 1542 struct ether_header *eh; 1543 struct virtio_net_hdr *hdr; 1544 struct virtio_net_hdr_mrg_rxbuf *mhdr; 1545 int len, deq, nbufs, adjsz, rx_npkts; 1546 1547 ifp = sc->vtnet_ifp; 1548 vq = sc->vtnet_rx_vq; 1549 hdr = &lhdr; 1550 deq = 0; 1551 rx_npkts = 0; 1552 1553 ASSERT_SERIALIZED(&sc->vtnet_slz); 1554 1555 while (--count >= 0) { 1556 m = virtqueue_dequeue(vq, &len); 1557 if (m == NULL) 1558 break; 1559 deq++; 1560 1561 if (len < sc->vtnet_hdr_size + ETHER_HDR_LEN) { 1562 ifp->if_ierrors++; 1563 vtnet_discard_rxbuf(sc, m); 1564 continue; 1565 } 1566 1567 if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) { 1568 nbufs = 1; 1569 adjsz = sizeof(struct vtnet_rx_header); 1570 /* 1571 * Account for our pad between the header and 1572 * the actual start of the frame. 1573 */ 1574 len += VTNET_RX_HEADER_PAD; 1575 } else { 1576 mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *); 1577 nbufs = mhdr->num_buffers; 1578 adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1579 } 1580 1581 if (vtnet_replace_rxbuf(sc, m, len) != 0) { 1582 ifp->if_iqdrops++; 1583 vtnet_discard_rxbuf(sc, m); 1584 if (nbufs > 1) 1585 vtnet_discard_merged_rxbuf(sc, nbufs); 1586 continue; 1587 } 1588 1589 m->m_pkthdr.len = len; 1590 m->m_pkthdr.rcvif = ifp; 1591 m->m_pkthdr.csum_flags = 0; 1592 1593 if (nbufs > 1) { 1594 if (vtnet_rxeof_merged(sc, m, nbufs) != 0) 1595 continue; 1596 } 1597 1598 ifp->if_ipackets++; 1599 1600 /* 1601 * Save copy of header before we strip it. For both mergeable 1602 * and non-mergeable, the VirtIO header is placed first in the 1603 * mbuf's data. We no longer need num_buffers, so always use a 1604 * virtio_net_hdr. 1605 */ 1606 memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr)); 1607 m_adj(m, adjsz); 1608 1609 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1610 eh = mtod(m, struct ether_header *); 1611 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { 1612 vtnet_vlan_tag_remove(m); 1613 1614 /* 1615 * With the 802.1Q header removed, update the 1616 * checksum starting location accordingly. 1617 */ 1618 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) 1619 hdr->csum_start -= 1620 ETHER_VLAN_ENCAP_LEN; 1621 } 1622 } 1623 1624 if (ifp->if_capenable & IFCAP_RXCSUM && 1625 hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 1626 if (vtnet_rx_csum(sc, m, hdr) != 0) 1627 sc->vtnet_stats.rx_csum_failed++; 1628 } 1629 1630 lwkt_serialize_exit(&sc->vtnet_slz); 1631 rx_npkts++; 1632 (*ifp->if_input)(ifp, m); 1633 lwkt_serialize_enter(&sc->vtnet_slz); 1634 1635 /* 1636 * The interface may have been stopped while we were 1637 * passing the packet up the network stack. 1638 */ 1639 if ((ifp->if_flags & IFF_RUNNING) == 0) 1640 break; 1641 } 1642 1643 virtqueue_notify(vq, &sc->vtnet_slz); 1644 1645 if (rx_npktsp != NULL) 1646 *rx_npktsp = rx_npkts; 1647 1648 return (count > 0 ? 0 : EAGAIN); 1649 } 1650 1651 static void 1652 vtnet_rx_intr_task(void *arg) 1653 { 1654 struct vtnet_softc *sc; 1655 struct ifnet *ifp; 1656 int more; 1657 1658 sc = arg; 1659 ifp = sc->vtnet_ifp; 1660 1661 // lwkt_serialize_enter(&sc->vtnet_slz); 1662 1663 if ((ifp->if_flags & IFF_RUNNING) == 0) { 1664 vtnet_enable_rx_intr(sc); 1665 // lwkt_serialize_exit(&sc->vtnet_slz); 1666 return; 1667 } 1668 1669 more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL); 1670 if (!more && vtnet_enable_rx_intr(sc) != 0) { 1671 vtnet_disable_rx_intr(sc); 1672 more = 1; 1673 } 1674 1675 // lwkt_serialize_exit(&sc->vtnet_slz); 1676 1677 if (more) { 1678 sc->vtnet_stats.rx_task_rescheduled++; 1679 /* XXX: Loop?! */ 1680 vtnet_rx_intr_task(sc); 1681 } 1682 } 1683 1684 static int 1685 vtnet_rx_vq_intr(void *xsc) 1686 { 1687 struct vtnet_softc *sc; 1688 1689 sc = xsc; 1690 1691 vtnet_disable_rx_intr(sc); 1692 vtnet_rx_intr_task(sc); 1693 1694 return (1); 1695 } 1696 1697 static void 1698 vtnet_txeof(struct vtnet_softc *sc) 1699 { 1700 struct virtqueue *vq; 1701 struct ifnet *ifp; 1702 struct vtnet_tx_header *txhdr; 1703 int deq; 1704 1705 vq = sc->vtnet_tx_vq; 1706 ifp = sc->vtnet_ifp; 1707 deq = 0; 1708 1709 ASSERT_SERIALIZED(&sc->vtnet_slz); 1710 1711 while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) { 1712 deq++; 1713 ifp->if_opackets++; 1714 m_freem(txhdr->vth_mbuf); 1715 } 1716 1717 if (deq > 0) { 1718 ifq_clr_oactive(&ifp->if_snd); 1719 if (virtqueue_empty(vq)) 1720 sc->vtnet_watchdog_timer = 0; 1721 } 1722 } 1723 1724 static struct mbuf * 1725 vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m, 1726 struct virtio_net_hdr *hdr) 1727 { 1728 struct ifnet *ifp; 1729 struct ether_header *eh; 1730 struct ether_vlan_header *evh; 1731 struct ip *ip; 1732 struct ip6_hdr *ip6; 1733 struct tcphdr *tcp; 1734 int ip_offset; 1735 uint16_t eth_type, csum_start; 1736 uint8_t ip_proto, gso_type; 1737 1738 ifp = sc->vtnet_ifp; 1739 M_ASSERTPKTHDR(m); 1740 1741 ip_offset = sizeof(struct ether_header); 1742 if (m->m_len < ip_offset) { 1743 if ((m = m_pullup(m, ip_offset)) == NULL) 1744 return (NULL); 1745 } 1746 1747 eh = mtod(m, struct ether_header *); 1748 eth_type = ntohs(eh->ether_type); 1749 if (eth_type == ETHERTYPE_VLAN) { 1750 ip_offset = sizeof(struct ether_vlan_header); 1751 if (m->m_len < ip_offset) { 1752 if ((m = m_pullup(m, ip_offset)) == NULL) 1753 return (NULL); 1754 } 1755 evh = mtod(m, struct ether_vlan_header *); 1756 eth_type = ntohs(evh->evl_proto); 1757 } 1758 1759 switch (eth_type) { 1760 case ETHERTYPE_IP: 1761 if (m->m_len < ip_offset + sizeof(struct ip)) { 1762 m = m_pullup(m, ip_offset + sizeof(struct ip)); 1763 if (m == NULL) 1764 return (NULL); 1765 } 1766 1767 ip = (struct ip *)(mtod(m, uint8_t *) + ip_offset); 1768 ip_proto = ip->ip_p; 1769 csum_start = ip_offset + (ip->ip_hl << 2); 1770 gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 1771 break; 1772 1773 case ETHERTYPE_IPV6: 1774 if (m->m_len < ip_offset + sizeof(struct ip6_hdr)) { 1775 m = m_pullup(m, ip_offset + sizeof(struct ip6_hdr)); 1776 if (m == NULL) 1777 return (NULL); 1778 } 1779 1780 ip6 = (struct ip6_hdr *)(mtod(m, uint8_t *) + ip_offset); 1781 /* 1782 * XXX Assume no extension headers are present. Presently, 1783 * this will always be true in the case of TSO, and FreeBSD 1784 * does not perform checksum offloading of IPv6 yet. 1785 */ 1786 ip_proto = ip6->ip6_nxt; 1787 csum_start = ip_offset + sizeof(struct ip6_hdr); 1788 gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 1789 break; 1790 1791 default: 1792 return (m); 1793 } 1794 1795 if (m->m_pkthdr.csum_flags & VTNET_CSUM_OFFLOAD) { 1796 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM; 1797 hdr->csum_start = csum_start; 1798 hdr->csum_offset = m->m_pkthdr.csum_data; 1799 1800 sc->vtnet_stats.tx_csum_offloaded++; 1801 } 1802 1803 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1804 if (ip_proto != IPPROTO_TCP) 1805 return (m); 1806 1807 if (m->m_len < csum_start + sizeof(struct tcphdr)) { 1808 m = m_pullup(m, csum_start + sizeof(struct tcphdr)); 1809 if (m == NULL) 1810 return (NULL); 1811 } 1812 1813 tcp = (struct tcphdr *)(mtod(m, uint8_t *) + csum_start); 1814 hdr->gso_type = gso_type; 1815 hdr->hdr_len = csum_start + (tcp->th_off << 2); 1816 hdr->gso_size = m->m_pkthdr.tso_segsz; 1817 1818 if (tcp->th_flags & TH_CWR) { 1819 /* 1820 * Drop if we did not negotiate VIRTIO_NET_F_HOST_ECN. 1821 * ECN support is only configurable globally with the 1822 * net.inet.tcp.ecn.enable sysctl knob. 1823 */ 1824 if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) { 1825 if_printf(ifp, "TSO with ECN not supported " 1826 "by host\n"); 1827 m_freem(m); 1828 return (NULL); 1829 } 1830 1831 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; 1832 } 1833 1834 sc->vtnet_stats.tx_tso_offloaded++; 1835 } 1836 1837 return (m); 1838 } 1839 1840 static int 1841 vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head, 1842 struct vtnet_tx_header *txhdr) 1843 { 1844 struct sglist sg; 1845 struct sglist_seg segs[VTNET_MAX_TX_SEGS]; 1846 struct virtqueue *vq; 1847 struct mbuf *m; 1848 int collapsed, error; 1849 1850 vq = sc->vtnet_tx_vq; 1851 m = *m_head; 1852 collapsed = 0; 1853 1854 sglist_init(&sg, VTNET_MAX_TX_SEGS, segs); 1855 error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size); 1856 KASSERT(error == 0 && sg.sg_nseg == 1, 1857 ("cannot add header to sglist")); 1858 1859 again: 1860 error = sglist_append_mbuf(&sg, m); 1861 if (error) { 1862 if (collapsed) 1863 goto fail; 1864 1865 //m = m_collapse(m, MB_DONTWAIT, VTNET_MAX_TX_SEGS - 1); 1866 m = m_defrag(m, MB_DONTWAIT); 1867 if (m == NULL) 1868 goto fail; 1869 1870 *m_head = m; 1871 collapsed = 1; 1872 goto again; 1873 } 1874 1875 txhdr->vth_mbuf = m; 1876 1877 return (virtqueue_enqueue(vq, txhdr, &sg, sg.sg_nseg, 0)); 1878 1879 fail: 1880 m_freem(*m_head); 1881 *m_head = NULL; 1882 1883 return (ENOBUFS); 1884 } 1885 1886 static struct mbuf * 1887 vtnet_vlan_tag_insert(struct mbuf *m) 1888 { 1889 struct mbuf *n; 1890 struct ether_vlan_header *evl; 1891 1892 if (M_WRITABLE(m) == 0) { 1893 n = m_dup(m, MB_DONTWAIT); 1894 m_freem(m); 1895 if ((m = n) == NULL) 1896 return (NULL); 1897 } 1898 1899 M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, MB_DONTWAIT); 1900 if (m == NULL) 1901 return (NULL); 1902 if (m->m_len < sizeof(struct ether_vlan_header)) { 1903 m = m_pullup(m, sizeof(struct ether_vlan_header)); 1904 if (m == NULL) 1905 return (NULL); 1906 } 1907 1908 /* Insert 802.1Q header into the existing Ethernet header. */ 1909 evl = mtod(m, struct ether_vlan_header *); 1910 bcopy((char *) evl + ETHER_VLAN_ENCAP_LEN, 1911 (char *) evl, ETHER_HDR_LEN - ETHER_TYPE_LEN); 1912 evl->evl_encap_proto = htons(ETHERTYPE_VLAN); 1913 evl->evl_tag = htons(m->m_pkthdr.ether_vlantag); 1914 m->m_flags &= ~M_VLANTAG; 1915 1916 return (m); 1917 } 1918 1919 static int 1920 vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head) 1921 { 1922 struct vtnet_tx_header *txhdr; 1923 struct virtio_net_hdr *hdr; 1924 struct mbuf *m; 1925 int error; 1926 1927 txhdr = &sc->vtnet_txhdrarea[sc->vtnet_txhdridx]; 1928 memset(txhdr, 0, sizeof(struct vtnet_tx_header)); 1929 1930 /* 1931 * Always use the non-mergeable header to simplify things. When 1932 * the mergeable feature is negotiated, the num_buffers field 1933 * must be set to zero. We use vtnet_hdr_size later to enqueue 1934 * the correct header size to the host. 1935 */ 1936 hdr = &txhdr->vth_uhdr.hdr; 1937 m = *m_head; 1938 1939 error = ENOBUFS; 1940 1941 if (m->m_flags & M_VLANTAG) { 1942 //m = ether_vlanencap(m, m->m_pkthdr.ether_vtag); 1943 m = vtnet_vlan_tag_insert(m); 1944 if ((*m_head = m) == NULL) 1945 goto fail; 1946 m->m_flags &= ~M_VLANTAG; 1947 } 1948 1949 if (m->m_pkthdr.csum_flags != 0) { 1950 m = vtnet_tx_offload(sc, m, hdr); 1951 if ((*m_head = m) == NULL) 1952 goto fail; 1953 } 1954 1955 error = vtnet_enqueue_txbuf(sc, m_head, txhdr); 1956 if (error == 0) 1957 sc->vtnet_txhdridx = 1958 (sc->vtnet_txhdridx + 1) % ((sc->vtnet_tx_size / 2) + 1); 1959 fail: 1960 return (error); 1961 } 1962 1963 static void 1964 vtnet_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 1965 { 1966 struct vtnet_softc *sc; 1967 1968 sc = ifp->if_softc; 1969 1970 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); 1971 lwkt_serialize_enter(&sc->vtnet_slz); 1972 vtnet_start_locked(ifp, ifsq); 1973 lwkt_serialize_exit(&sc->vtnet_slz); 1974 } 1975 1976 static void 1977 vtnet_start_locked(struct ifnet *ifp, struct ifaltq_subque *ifsq) 1978 { 1979 struct vtnet_softc *sc; 1980 struct virtqueue *vq; 1981 struct mbuf *m0; 1982 int enq; 1983 1984 sc = ifp->if_softc; 1985 vq = sc->vtnet_tx_vq; 1986 enq = 0; 1987 1988 ASSERT_SERIALIZED(&sc->vtnet_slz); 1989 1990 if ((ifp->if_flags & (IFF_RUNNING)) != 1991 IFF_RUNNING || ((sc->vtnet_flags & VTNET_FLAG_LINK) == 0)) 1992 return; 1993 1994 #ifdef VTNET_TX_INTR_MODERATION 1995 if (virtqueue_nused(vq) >= sc->vtnet_tx_size / 2) 1996 vtnet_txeof(sc); 1997 #endif 1998 1999 while (!ifsq_is_empty(ifsq)) { 2000 if (virtqueue_full(vq)) { 2001 ifq_set_oactive(&ifp->if_snd); 2002 break; 2003 } 2004 2005 m0 = ifq_dequeue(&ifp->if_snd); 2006 if (m0 == NULL) 2007 break; 2008 2009 if (vtnet_encap(sc, &m0) != 0) { 2010 if (m0 == NULL) 2011 break; 2012 ifq_prepend(&ifp->if_snd, m0); 2013 ifq_set_oactive(&ifp->if_snd); 2014 break; 2015 } 2016 2017 enq++; 2018 ETHER_BPF_MTAP(ifp, m0); 2019 } 2020 2021 if (enq > 0) { 2022 virtqueue_notify(vq, &sc->vtnet_slz); 2023 sc->vtnet_watchdog_timer = VTNET_WATCHDOG_TIMEOUT; 2024 } 2025 } 2026 2027 static void 2028 vtnet_tick(void *xsc) 2029 { 2030 struct vtnet_softc *sc; 2031 2032 sc = xsc; 2033 2034 #if 0 2035 ASSERT_SERIALIZED(&sc->vtnet_slz); 2036 #ifdef VTNET_DEBUG 2037 virtqueue_dump(sc->vtnet_rx_vq); 2038 virtqueue_dump(sc->vtnet_tx_vq); 2039 #endif 2040 2041 vtnet_watchdog(sc); 2042 callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); 2043 #endif 2044 } 2045 2046 static void 2047 vtnet_tx_intr_task(void *arg) 2048 { 2049 struct vtnet_softc *sc; 2050 struct ifnet *ifp; 2051 struct ifaltq_subque *ifsq; 2052 2053 sc = arg; 2054 ifp = sc->vtnet_ifp; 2055 ifsq = ifq_get_subq_default(&ifp->if_snd); 2056 2057 // lwkt_serialize_enter(&sc->vtnet_slz); 2058 2059 if ((ifp->if_flags & IFF_RUNNING) == 0) { 2060 vtnet_enable_tx_intr(sc); 2061 // lwkt_serialize_exit(&sc->vtnet_slz); 2062 return; 2063 } 2064 2065 vtnet_txeof(sc); 2066 2067 if (!ifsq_is_empty(ifsq)) 2068 vtnet_start_locked(ifp, ifsq); 2069 2070 if (vtnet_enable_tx_intr(sc) != 0) { 2071 vtnet_disable_tx_intr(sc); 2072 sc->vtnet_stats.tx_task_rescheduled++; 2073 // lwkt_serialize_exit(&sc->vtnet_slz); 2074 vtnet_tx_intr_task(sc); 2075 /* XXX: loop?! */ 2076 return; 2077 } 2078 2079 // lwkt_serialize_exit(&sc->vtnet_slz); 2080 } 2081 2082 static int 2083 vtnet_tx_vq_intr(void *xsc) 2084 { 2085 struct vtnet_softc *sc; 2086 2087 sc = xsc; 2088 2089 vtnet_disable_tx_intr(sc); 2090 vtnet_tx_intr_task(sc); 2091 2092 return (1); 2093 } 2094 2095 static void 2096 vtnet_stop(struct vtnet_softc *sc) 2097 { 2098 device_t dev; 2099 struct ifnet *ifp; 2100 2101 dev = sc->vtnet_dev; 2102 ifp = sc->vtnet_ifp; 2103 2104 ASSERT_SERIALIZED(&sc->vtnet_slz); 2105 2106 sc->vtnet_watchdog_timer = 0; 2107 callout_stop(&sc->vtnet_tick_ch); 2108 ifq_clr_oactive(&ifp->if_snd); 2109 ifp->if_flags &= ~(IFF_RUNNING); 2110 2111 vtnet_disable_rx_intr(sc); 2112 vtnet_disable_tx_intr(sc); 2113 2114 /* 2115 * Stop the host VirtIO adapter. Note this will reset the host 2116 * adapter's state back to the pre-initialized state, so in 2117 * order to make the device usable again, we must drive it 2118 * through virtio_reinit() and virtio_reinit_complete(). 2119 */ 2120 virtio_stop(dev); 2121 2122 sc->vtnet_flags &= ~VTNET_FLAG_LINK; 2123 2124 vtnet_free_rx_mbufs(sc); 2125 vtnet_free_tx_mbufs(sc); 2126 } 2127 2128 static int 2129 vtnet_reinit(struct vtnet_softc *sc) 2130 { 2131 struct ifnet *ifp; 2132 uint64_t features; 2133 2134 ifp = sc->vtnet_ifp; 2135 features = sc->vtnet_features; 2136 2137 /* 2138 * Re-negotiate with the host, removing any disabled receive 2139 * features. Transmit features are disabled only on our side 2140 * via if_capenable and if_hwassist. 2141 */ 2142 2143 if (ifp->if_capabilities & IFCAP_RXCSUM) { 2144 if ((ifp->if_capenable & IFCAP_RXCSUM) == 0) 2145 features &= ~VIRTIO_NET_F_GUEST_CSUM; 2146 } 2147 2148 if (ifp->if_capabilities & IFCAP_LRO) { 2149 if ((ifp->if_capenable & IFCAP_LRO) == 0) 2150 features &= ~VTNET_LRO_FEATURES; 2151 } 2152 2153 if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) { 2154 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0) 2155 features &= ~VIRTIO_NET_F_CTRL_VLAN; 2156 } 2157 2158 return (virtio_reinit(sc->vtnet_dev, features)); 2159 } 2160 2161 static void 2162 vtnet_init_locked(struct vtnet_softc *sc) 2163 { 2164 device_t dev; 2165 struct ifnet *ifp; 2166 int error; 2167 2168 dev = sc->vtnet_dev; 2169 ifp = sc->vtnet_ifp; 2170 2171 ASSERT_SERIALIZED(&sc->vtnet_slz); 2172 2173 if (ifp->if_flags & IFF_RUNNING) 2174 return; 2175 2176 /* Stop host's adapter, cancel any pending I/O. */ 2177 vtnet_stop(sc); 2178 2179 /* Reinitialize the host device. */ 2180 error = vtnet_reinit(sc); 2181 if (error) { 2182 device_printf(dev, 2183 "reinitialization failed, stopping device...\n"); 2184 vtnet_stop(sc); 2185 return; 2186 } 2187 2188 /* Update host with assigned MAC address. */ 2189 bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN); 2190 vtnet_set_hwaddr(sc); 2191 2192 ifp->if_hwassist = 0; 2193 if (ifp->if_capenable & IFCAP_TXCSUM) 2194 ifp->if_hwassist |= VTNET_CSUM_OFFLOAD; 2195 if (ifp->if_capenable & IFCAP_TSO4) 2196 ifp->if_hwassist |= CSUM_TSO; 2197 2198 error = vtnet_init_rx_vq(sc); 2199 if (error) { 2200 device_printf(dev, 2201 "cannot allocate mbufs for Rx virtqueue\n"); 2202 vtnet_stop(sc); 2203 return; 2204 } 2205 2206 if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) { 2207 if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) { 2208 /* Restore promiscuous and all-multicast modes. */ 2209 vtnet_rx_filter(sc); 2210 2211 /* Restore filtered MAC addresses. */ 2212 vtnet_rx_filter_mac(sc); 2213 } 2214 2215 /* Restore VLAN filters. */ 2216 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 2217 vtnet_rx_filter_vlan(sc); 2218 } 2219 2220 { 2221 vtnet_enable_rx_intr(sc); 2222 vtnet_enable_tx_intr(sc); 2223 } 2224 2225 ifp->if_flags |= IFF_RUNNING; 2226 ifq_clr_oactive(&ifp->if_snd); 2227 2228 virtio_reinit_complete(dev); 2229 2230 vtnet_update_link_status(sc); 2231 callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc); 2232 } 2233 2234 static void 2235 vtnet_init(void *xsc) 2236 { 2237 struct vtnet_softc *sc; 2238 2239 sc = xsc; 2240 2241 lwkt_serialize_enter(&sc->vtnet_slz); 2242 vtnet_init_locked(sc); 2243 lwkt_serialize_exit(&sc->vtnet_slz); 2244 } 2245 2246 static void 2247 vtnet_exec_ctrl_cmd(struct vtnet_softc *sc, void *cookie, 2248 struct sglist *sg, int readable, int writable) 2249 { 2250 struct virtqueue *vq; 2251 void *c; 2252 2253 vq = sc->vtnet_ctrl_vq; 2254 2255 ASSERT_SERIALIZED(&sc->vtnet_slz); 2256 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ, 2257 ("no control virtqueue")); 2258 KASSERT(virtqueue_empty(vq), 2259 ("control command already enqueued")); 2260 2261 if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0) 2262 return; 2263 2264 virtqueue_notify(vq, &sc->vtnet_slz); 2265 2266 /* 2267 * Poll until the command is complete. Previously, we would 2268 * sleep until the control virtqueue interrupt handler woke 2269 * us up, but dropping the VTNET_MTX leads to serialization 2270 * difficulties. 2271 * 2272 * Furthermore, it appears QEMU/KVM only allocates three MSIX 2273 * vectors. Two of those vectors are needed for the Rx and Tx 2274 * virtqueues. We do not support sharing both a Vq and config 2275 * changed notification on the same MSIX vector. 2276 */ 2277 c = virtqueue_poll(vq, NULL); 2278 KASSERT(c == cookie, ("unexpected control command response")); 2279 } 2280 2281 static void 2282 vtnet_rx_filter(struct vtnet_softc *sc) 2283 { 2284 device_t dev; 2285 struct ifnet *ifp; 2286 2287 dev = sc->vtnet_dev; 2288 ifp = sc->vtnet_ifp; 2289 2290 ASSERT_SERIALIZED(&sc->vtnet_slz); 2291 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 2292 ("CTRL_RX feature not negotiated")); 2293 2294 if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0) 2295 device_printf(dev, "cannot %s promiscuous mode\n", 2296 ifp->if_flags & IFF_PROMISC ? "enable" : "disable"); 2297 2298 if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0) 2299 device_printf(dev, "cannot %s all-multicast mode\n", 2300 ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable"); 2301 } 2302 2303 static int 2304 vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on) 2305 { 2306 struct virtio_net_ctrl_hdr hdr __aligned(2); 2307 struct sglist_seg segs[3]; 2308 struct sglist sg; 2309 uint8_t onoff, ack; 2310 int error; 2311 2312 if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) 2313 return (ENOTSUP); 2314 2315 error = 0; 2316 2317 hdr.class = VIRTIO_NET_CTRL_RX; 2318 hdr.cmd = cmd; 2319 onoff = !!on; 2320 ack = VIRTIO_NET_ERR; 2321 2322 sglist_init(&sg, 3, segs); 2323 error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 2324 error |= sglist_append(&sg, &onoff, sizeof(uint8_t)); 2325 error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 2326 KASSERT(error == 0 && sg.sg_nseg == 3, 2327 ("error adding Rx filter message to sglist")); 2328 2329 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 2330 2331 return (ack == VIRTIO_NET_OK ? 0 : EIO); 2332 } 2333 2334 static int 2335 vtnet_set_promisc(struct vtnet_softc *sc, int on) 2336 { 2337 2338 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on)); 2339 } 2340 2341 static int 2342 vtnet_set_allmulti(struct vtnet_softc *sc, int on) 2343 { 2344 2345 return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on)); 2346 } 2347 2348 static void 2349 vtnet_rx_filter_mac(struct vtnet_softc *sc) 2350 { 2351 struct virtio_net_ctrl_hdr hdr __aligned(2); 2352 struct vtnet_mac_filter *filter; 2353 struct sglist_seg segs[4]; 2354 struct sglist sg; 2355 struct ifnet *ifp; 2356 struct ifaddr *ifa; 2357 struct ifaddr_container *ifac; 2358 struct ifmultiaddr *ifma; 2359 int ucnt, mcnt, promisc, allmulti, error; 2360 uint8_t ack; 2361 2362 ifp = sc->vtnet_ifp; 2363 ucnt = 0; 2364 mcnt = 0; 2365 promisc = 0; 2366 allmulti = 0; 2367 error = 0; 2368 2369 ASSERT_SERIALIZED(&sc->vtnet_slz); 2370 KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX, 2371 ("CTRL_RX feature not negotiated")); 2372 2373 /* Use the MAC filtering table allocated in vtnet_attach. */ 2374 filter = sc->vtnet_macfilter; 2375 memset(filter, 0, sizeof(struct vtnet_mac_filter)); 2376 2377 /* Unicast MAC addresses: */ 2378 //if_addr_rlock(ifp); 2379 TAILQ_FOREACH(ifac, &ifp->if_addrheads[mycpuid], ifa_link) { 2380 ifa = ifac->ifa; 2381 if (ifa->ifa_addr->sa_family != AF_LINK) 2382 continue; 2383 else if (ucnt == VTNET_MAX_MAC_ENTRIES) 2384 break; 2385 2386 bcopy(LLADDR((struct sockaddr_dl *)ifa->ifa_addr), 2387 &filter->vmf_unicast.macs[ucnt], ETHER_ADDR_LEN); 2388 ucnt++; 2389 } 2390 //if_addr_runlock(ifp); 2391 2392 if (ucnt >= VTNET_MAX_MAC_ENTRIES) { 2393 promisc = 1; 2394 filter->vmf_unicast.nentries = 0; 2395 2396 if_printf(ifp, "more than %d MAC addresses assigned, " 2397 "falling back to promiscuous mode\n", 2398 VTNET_MAX_MAC_ENTRIES); 2399 } else 2400 filter->vmf_unicast.nentries = ucnt; 2401 2402 /* Multicast MAC addresses: */ 2403 //if_maddr_rlock(ifp); 2404 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2405 if (ifma->ifma_addr->sa_family != AF_LINK) 2406 continue; 2407 else if (mcnt == VTNET_MAX_MAC_ENTRIES) 2408 break; 2409 2410 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2411 &filter->vmf_multicast.macs[mcnt], ETHER_ADDR_LEN); 2412 mcnt++; 2413 } 2414 //if_maddr_runlock(ifp); 2415 2416 if (mcnt >= VTNET_MAX_MAC_ENTRIES) { 2417 allmulti = 1; 2418 filter->vmf_multicast.nentries = 0; 2419 2420 if_printf(ifp, "more than %d multicast MAC addresses " 2421 "assigned, falling back to all-multicast mode\n", 2422 VTNET_MAX_MAC_ENTRIES); 2423 } else 2424 filter->vmf_multicast.nentries = mcnt; 2425 2426 if (promisc && allmulti) 2427 goto out; 2428 2429 hdr.class = VIRTIO_NET_CTRL_MAC; 2430 hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 2431 ack = VIRTIO_NET_ERR; 2432 2433 sglist_init(&sg, 4, segs); 2434 error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 2435 error |= sglist_append(&sg, &filter->vmf_unicast, 2436 sizeof(struct vtnet_mac_table)); 2437 error |= sglist_append(&sg, &filter->vmf_multicast, 2438 sizeof(struct vtnet_mac_table)); 2439 error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 2440 KASSERT(error == 0 && sg.sg_nseg == 4, 2441 ("error adding MAC filtering message to sglist")); 2442 2443 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 2444 2445 if (ack != VIRTIO_NET_OK) 2446 if_printf(ifp, "error setting host MAC filter table\n"); 2447 2448 out: 2449 if (promisc) 2450 if (vtnet_set_promisc(sc, 1) != 0) 2451 if_printf(ifp, "cannot enable promiscuous mode\n"); 2452 if (allmulti) 2453 if (vtnet_set_allmulti(sc, 1) != 0) 2454 if_printf(ifp, "cannot enable all-multicast mode\n"); 2455 } 2456 2457 static int 2458 vtnet_exec_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 2459 { 2460 struct virtio_net_ctrl_hdr hdr __aligned(2); 2461 struct sglist_seg segs[3]; 2462 struct sglist sg; 2463 uint8_t ack; 2464 int error; 2465 2466 hdr.class = VIRTIO_NET_CTRL_VLAN; 2467 hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 2468 ack = VIRTIO_NET_ERR; 2469 error = 0; 2470 2471 sglist_init(&sg, 3, segs); 2472 error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr)); 2473 error |= sglist_append(&sg, &tag, sizeof(uint16_t)); 2474 error |= sglist_append(&sg, &ack, sizeof(uint8_t)); 2475 KASSERT(error == 0 && sg.sg_nseg == 3, 2476 ("error adding VLAN control message to sglist")); 2477 2478 vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1); 2479 2480 return (ack == VIRTIO_NET_OK ? 0 : EIO); 2481 } 2482 2483 static void 2484 vtnet_rx_filter_vlan(struct vtnet_softc *sc) 2485 { 2486 device_t dev; 2487 uint32_t w, mask; 2488 uint16_t tag; 2489 int i, nvlans, error; 2490 2491 ASSERT_SERIALIZED(&sc->vtnet_slz); 2492 KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER, 2493 ("VLAN_FILTER feature not negotiated")); 2494 2495 dev = sc->vtnet_dev; 2496 nvlans = sc->vtnet_nvlans; 2497 error = 0; 2498 2499 /* Enable filtering for each configured VLAN. */ 2500 for (i = 0; i < VTNET_VLAN_SHADOW_SIZE && nvlans > 0; i++) { 2501 w = sc->vtnet_vlan_shadow[i]; 2502 for (mask = 1, tag = i * 32; w != 0; mask <<= 1, tag++) { 2503 if ((w & mask) != 0) { 2504 w &= ~mask; 2505 nvlans--; 2506 if (vtnet_exec_vlan_filter(sc, 1, tag) != 0) 2507 error++; 2508 } 2509 } 2510 } 2511 2512 KASSERT(nvlans == 0, ("VLAN count incorrect")); 2513 if (error) 2514 device_printf(dev, "cannot restore VLAN filter table\n"); 2515 } 2516 2517 static void 2518 vtnet_set_vlan_filter(struct vtnet_softc *sc, int add, uint16_t tag) 2519 { 2520 struct ifnet *ifp; 2521 int idx, bit; 2522 2523 KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER, 2524 ("VLAN_FILTER feature not negotiated")); 2525 2526 if ((tag == 0) || (tag > 4095)) 2527 return; 2528 2529 ifp = sc->vtnet_ifp; 2530 idx = (tag >> 5) & 0x7F; 2531 bit = tag & 0x1F; 2532 2533 lwkt_serialize_enter(&sc->vtnet_slz); 2534 2535 /* Update shadow VLAN table. */ 2536 if (add) { 2537 sc->vtnet_nvlans++; 2538 sc->vtnet_vlan_shadow[idx] |= (1 << bit); 2539 } else { 2540 sc->vtnet_nvlans--; 2541 sc->vtnet_vlan_shadow[idx] &= ~(1 << bit); 2542 } 2543 2544 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 2545 if (vtnet_exec_vlan_filter(sc, add, tag) != 0) { 2546 device_printf(sc->vtnet_dev, 2547 "cannot %s VLAN %d %s the host filter table\n", 2548 add ? "add" : "remove", tag, 2549 add ? "to" : "from"); 2550 } 2551 } 2552 2553 lwkt_serialize_exit(&sc->vtnet_slz); 2554 } 2555 2556 static void 2557 vtnet_register_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 2558 { 2559 2560 if (ifp->if_softc != arg) 2561 return; 2562 2563 vtnet_set_vlan_filter(arg, 1, tag); 2564 } 2565 2566 static void 2567 vtnet_unregister_vlan(void *arg, struct ifnet *ifp, uint16_t tag) 2568 { 2569 2570 if (ifp->if_softc != arg) 2571 return; 2572 2573 vtnet_set_vlan_filter(arg, 0, tag); 2574 } 2575 2576 static int 2577 vtnet_ifmedia_upd(struct ifnet *ifp) 2578 { 2579 struct vtnet_softc *sc; 2580 struct ifmedia *ifm; 2581 2582 sc = ifp->if_softc; 2583 ifm = &sc->vtnet_media; 2584 2585 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2586 return (EINVAL); 2587 2588 return (0); 2589 } 2590 2591 static void 2592 vtnet_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) 2593 { 2594 struct vtnet_softc *sc; 2595 2596 sc = ifp->if_softc; 2597 2598 ifmr->ifm_status = IFM_AVALID; 2599 ifmr->ifm_active = IFM_ETHER; 2600 2601 lwkt_serialize_enter(&sc->vtnet_slz); 2602 if (vtnet_is_link_up(sc) != 0) { 2603 ifmr->ifm_status |= IFM_ACTIVE; 2604 ifmr->ifm_active |= VTNET_MEDIATYPE; 2605 } else 2606 ifmr->ifm_active |= IFM_NONE; 2607 lwkt_serialize_exit(&sc->vtnet_slz); 2608 } 2609 2610 static void 2611 vtnet_add_statistics(struct vtnet_softc *sc) 2612 { 2613 device_t dev; 2614 struct vtnet_statistics *stats; 2615 //struct sysctl_ctx_list *ctx; 2616 //struct sysctl_oid *tree; 2617 //struct sysctl_oid_list *child; 2618 int error = 0; 2619 2620 dev = sc->vtnet_dev; 2621 stats = &sc->vtnet_stats; 2622 sysctl_ctx_init(&sc->vtnet_sysctl_ctx); 2623 sc->vtnet_sysctl_tree = SYSCTL_ADD_NODE(&sc->vtnet_sysctl_ctx, 2624 SYSCTL_STATIC_CHILDREN(_hw), 2625 OID_AUTO, 2626 device_get_nameunit(dev), 2627 CTLFLAG_RD, 0, ""); 2628 2629 if (sc->vtnet_sysctl_tree == NULL) { 2630 device_printf(dev, "can't add sysctl node\n"); 2631 error = ENXIO; 2632 } 2633 2634 2635 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, 2636 SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, 2637 "mbuf_alloc_failed", CTLFLAG_RD, &stats->mbuf_alloc_failed, 2638 "Mbuf cluster allocation failures"); 2639 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, 2640 SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, 2641 "rx_frame_too_large", CTLFLAG_RD, &stats->rx_frame_too_large, 2642 "Received frame larger than the mbuf chain"); 2643 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx,SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_enq_replacement_failed", 2644 CTLFLAG_RD, &stats->rx_enq_replacement_failed, 2645 "Enqueuing the replacement receive mbuf failed"); 2646 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_mergeable_failed", 2647 CTLFLAG_RD, &stats->rx_mergeable_failed, 2648 "Mergeable buffers receive failures"); 2649 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ethtype", 2650 CTLFLAG_RD, &stats->rx_csum_bad_ethtype, 2651 "Received checksum offloaded buffer with unsupported " 2652 "Ethernet type"); 2653 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_start", 2654 CTLFLAG_RD, &stats->rx_csum_bad_start, 2655 "Received checksum offloaded buffer with incorrect start offset"); 2656 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_ipproto", 2657 CTLFLAG_RD, &stats->rx_csum_bad_ipproto, 2658 "Received checksum offloaded buffer with incorrect IP protocol"); 2659 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_bad_offset", 2660 CTLFLAG_RD, &stats->rx_csum_bad_offset, 2661 "Received checksum offloaded buffer with incorrect offset"); 2662 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_failed", 2663 CTLFLAG_RD, &stats->rx_csum_failed, 2664 "Received buffer checksum offload failed"); 2665 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_csum_offloaded", 2666 CTLFLAG_RD, &stats->rx_csum_offloaded, 2667 "Received buffer checksum offload succeeded"); 2668 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "rx_task_rescheduled", 2669 CTLFLAG_RD, &stats->rx_task_rescheduled, 2670 "Times the receive interrupt task rescheduled itself"); 2671 2672 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_offloaded", 2673 CTLFLAG_RD, &stats->tx_csum_offloaded, 2674 "Offloaded checksum of transmitted buffer"); 2675 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_offloaded", 2676 CTLFLAG_RD, &stats->tx_tso_offloaded, 2677 "Segmentation offload of transmitted buffer"); 2678 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_csum_bad_ethtype", 2679 CTLFLAG_RD, &stats->tx_csum_bad_ethtype, 2680 "Aborted transmit of checksum offloaded buffer with unknown " 2681 "Ethernet type"); 2682 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_tso_bad_ethtype", 2683 CTLFLAG_RD, &stats->tx_tso_bad_ethtype, 2684 "Aborted transmit of TSO buffer with unknown Ethernet type"); 2685 SYSCTL_ADD_ULONG(&sc->vtnet_sysctl_ctx, SYSCTL_CHILDREN(sc->vtnet_sysctl_tree), OID_AUTO, "tx_task_rescheduled", 2686 CTLFLAG_RD, &stats->tx_task_rescheduled, 2687 "Times the transmit interrupt task rescheduled itself"); 2688 } 2689 2690 static int 2691 vtnet_enable_rx_intr(struct vtnet_softc *sc) 2692 { 2693 2694 return (virtqueue_enable_intr(sc->vtnet_rx_vq)); 2695 } 2696 2697 static void 2698 vtnet_disable_rx_intr(struct vtnet_softc *sc) 2699 { 2700 2701 virtqueue_disable_intr(sc->vtnet_rx_vq); 2702 } 2703 2704 static int 2705 vtnet_enable_tx_intr(struct vtnet_softc *sc) 2706 { 2707 2708 #ifdef VTNET_TX_INTR_MODERATION 2709 return (0); 2710 #else 2711 return (virtqueue_enable_intr(sc->vtnet_tx_vq)); 2712 #endif 2713 } 2714 2715 static void 2716 vtnet_disable_tx_intr(struct vtnet_softc *sc) 2717 { 2718 2719 virtqueue_disable_intr(sc->vtnet_tx_vq); 2720 } 2721