1 /* $OpenBSD: if_vio.c,v 1.68 2025/01/28 19:53:06 sf Exp $ */ 2 3 /* 4 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg. 5 * Copyright (c) 2010 Minoura Makoto. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include "bpfilter.h" 30 #include "vlan.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/device.h> 35 #include <sys/intrmap.h> 36 #include <sys/mbuf.h> 37 #include <sys/mutex.h> 38 #include <sys/percpu.h> /* for CACHELINESIZE */ 39 #include <sys/sockio.h> 40 #include <sys/timeout.h> 41 42 #include <dev/pv/virtioreg.h> 43 #include <dev/pv/virtiovar.h> 44 45 #include <net/if.h> 46 #include <net/if_media.h> 47 #include <net/route.h> 48 49 #include <netinet/if_ether.h> 50 #include <netinet/tcp.h> 51 #include <netinet/tcp_timer.h> 52 #include <netinet/tcp_var.h> 53 #include <netinet/udp.h> 54 55 #if NBPFILTER > 0 56 #include <net/bpf.h> 57 #endif 58 59 #if VIRTIO_DEBUG 60 #define DPRINTF(x...) printf(x) 61 #else 62 #define DPRINTF(x...) 63 #endif 64 65 /* 66 * if_vioreg.h: 67 */ 68 /* Configuration registers */ 69 #define VIRTIO_NET_CONFIG_MAC 0 /* 8 bit x 6 byte */ 70 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16 bit */ 71 #define VIRTIO_NET_CONFIG_MAX_QUEUES 8 /* 16 bit */ 72 #define VIRTIO_NET_CONFIG_MTU 10 /* 16 bit */ 73 #define VIRTIO_NET_CONFIG_SPEED 12 /* 32 bit */ 74 #define VIRTIO_NET_CONFIG_DUPLEX 16 /* 8 bit */ 75 #define VIRTIO_NET_CONFIG_RSS_SIZE 17 /* 8 bit */ 76 #define VIRTIO_NET_CONFIG_RSS_LEN 18 /* 16 bit */ 77 #define VIRTIO_NET_CONFIG_HASH_TYPES 20 /* 16 bit */ 78 79 /* Feature bits */ 80 #define VIRTIO_NET_F_CSUM (1ULL<<0) 81 #define VIRTIO_NET_F_GUEST_CSUM (1ULL<<1) 82 #define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS (1ULL<<2) 83 #define VIRTIO_NET_F_MTU (1ULL<<3) 84 #define VIRTIO_NET_F_MAC (1ULL<<5) 85 #define VIRTIO_NET_F_GSO (1ULL<<6) 86 #define VIRTIO_NET_F_GUEST_TSO4 (1ULL<<7) 87 #define VIRTIO_NET_F_GUEST_TSO6 (1ULL<<8) 88 #define VIRTIO_NET_F_GUEST_ECN (1ULL<<9) 89 #define VIRTIO_NET_F_GUEST_UFO (1ULL<<10) 90 #define VIRTIO_NET_F_HOST_TSO4 (1ULL<<11) 91 #define VIRTIO_NET_F_HOST_TSO6 (1ULL<<12) 92 #define VIRTIO_NET_F_HOST_ECN (1ULL<<13) 93 #define VIRTIO_NET_F_HOST_UFO (1ULL<<14) 94 #define VIRTIO_NET_F_MRG_RXBUF (1ULL<<15) 95 #define VIRTIO_NET_F_STATUS (1ULL<<16) 96 #define VIRTIO_NET_F_CTRL_VQ (1ULL<<17) 97 #define VIRTIO_NET_F_CTRL_RX (1ULL<<18) 98 #define VIRTIO_NET_F_CTRL_VLAN (1ULL<<19) 99 #define VIRTIO_NET_F_CTRL_RX_EXTRA (1ULL<<20) 100 #define VIRTIO_NET_F_GUEST_ANNOUNCE (1ULL<<21) 101 #define VIRTIO_NET_F_MQ (1ULL<<22) 102 #define VIRTIO_NET_F_CTRL_MAC_ADDR (1ULL<<23) 103 #define VIRTIO_NET_F_HOST_USO (1ULL<<56) 104 #define VIRTIO_NET_F_HASH_REPORT (1ULL<<57) 105 #define VIRTIO_NET_F_GUEST_HDRLEN (1ULL<<59) 106 #define VIRTIO_NET_F_RSS (1ULL<<60) 107 #define VIRTIO_NET_F_RSC_EXT (1ULL<<61) 108 #define VIRTIO_NET_F_STANDBY (1ULL<<62) 109 #define VIRTIO_NET_F_SPEED_DUPLEX (1ULL<<63) 110 /* 111 * Config(8) flags. The lowest byte is reserved for generic virtio stuff. 112 */ 113 114 /* Workaround for vlan related bug in qemu < version 2.0 */ 115 #define CONFFLAG_QEMU_VLAN_BUG (1<<8) 116 117 static const struct virtio_feature_name virtio_net_feature_names[] = { 118 #if VIRTIO_DEBUG 119 { VIRTIO_NET_F_CSUM, "CSum" }, 120 { VIRTIO_NET_F_GUEST_CSUM, "GuestCSum" }, 121 { VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, "CtrlGuestOffl" }, 122 { VIRTIO_NET_F_MTU, "MTU", }, 123 { VIRTIO_NET_F_MAC, "MAC" }, 124 { VIRTIO_NET_F_GSO, "GSO" }, 125 { VIRTIO_NET_F_GUEST_TSO4, "GuestTSO4" }, 126 { VIRTIO_NET_F_GUEST_TSO6, "GuestTSO6" }, 127 { VIRTIO_NET_F_GUEST_ECN, "GuestECN" }, 128 { VIRTIO_NET_F_GUEST_UFO, "GuestUFO" }, 129 { VIRTIO_NET_F_HOST_TSO4, "HostTSO4" }, 130 { VIRTIO_NET_F_HOST_TSO6, "HostTSO6" }, 131 { VIRTIO_NET_F_HOST_ECN, "HostECN" }, 132 { VIRTIO_NET_F_HOST_UFO, "HostUFO" }, 133 { VIRTIO_NET_F_MRG_RXBUF, "MrgRXBuf" }, 134 { VIRTIO_NET_F_STATUS, "Status" }, 135 { VIRTIO_NET_F_CTRL_VQ, "CtrlVQ" }, 136 { VIRTIO_NET_F_CTRL_RX, "CtrlRX" }, 137 { VIRTIO_NET_F_CTRL_VLAN, "CtrlVLAN" }, 138 { VIRTIO_NET_F_CTRL_RX_EXTRA, "CtrlRXExtra" }, 139 { VIRTIO_NET_F_GUEST_ANNOUNCE, "GuestAnnounce" }, 140 { VIRTIO_NET_F_MQ, "MQ" }, 141 { VIRTIO_NET_F_CTRL_MAC_ADDR, "CtrlMAC" }, 142 { VIRTIO_NET_F_HOST_USO, "HostUso" }, 143 { VIRTIO_NET_F_HASH_REPORT, "HashRpt" }, 144 { VIRTIO_NET_F_GUEST_HDRLEN, "GuestHdrlen" }, 145 { VIRTIO_NET_F_RSS, "RSS" }, 146 { VIRTIO_NET_F_RSC_EXT, "RSSExt" }, 147 { VIRTIO_NET_F_STANDBY, "Stdby" }, 148 { VIRTIO_NET_F_SPEED_DUPLEX, "SpdDplx" }, 149 #endif 150 { 0, NULL } 151 }; 152 153 /* Status */ 154 #define VIRTIO_NET_S_LINK_UP 1 155 156 /* Packet header structure */ 157 struct virtio_net_hdr { 158 uint8_t flags; 159 uint8_t gso_type; 160 uint16_t hdr_len; 161 uint16_t gso_size; 162 uint16_t csum_start; 163 uint16_t csum_offset; 164 165 /* only present if VIRTIO_NET_F_MRG_RXBUF is negotiated */ 166 uint16_t num_buffers; 167 } __packed; 168 169 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 170 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /* flags */ 171 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 172 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 173 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 174 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 175 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 176 177 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 178 179 /* Control virtqueue */ 180 struct virtio_net_ctrl_cmd { 181 uint8_t class; 182 uint8_t command; 183 } __packed; 184 #define VIRTIO_NET_CTRL_RX 0 185 # define VIRTIO_NET_CTRL_RX_PROMISC 0 186 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 187 188 #define VIRTIO_NET_CTRL_MAC 1 189 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 190 191 #define VIRTIO_NET_CTRL_VLAN 2 192 # define VIRTIO_NET_CTRL_VLAN_ADD 0 193 # define VIRTIO_NET_CTRL_VLAN_DEL 1 194 195 #define VIRTIO_NET_CTRL_MQ 4 196 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 197 # define VIRTIO_NET_CTRL_MQ_RSS_CONFIG 1 198 # define VIRTIO_NET_CTRL_MQ_HASH_CONFIG 2 199 200 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5 201 # define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0 202 203 struct virtio_net_ctrl_status { 204 uint8_t ack; 205 } __packed; 206 #define VIRTIO_NET_OK 0 207 #define VIRTIO_NET_ERR 1 208 209 struct virtio_net_ctrl_rx { 210 uint8_t onoff; 211 } __packed; 212 213 struct virtio_net_ctrl_mq_pairs_set { 214 uint16_t virtqueue_pairs; 215 }; 216 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 217 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 218 219 struct virtio_net_ctrl_guest_offloads { 220 uint64_t offloads; 221 } __packed; 222 223 struct virtio_net_ctrl_mac_tbl { 224 uint32_t nentries; 225 uint8_t macs[][ETHER_ADDR_LEN]; 226 } __packed; 227 228 struct virtio_net_ctrl_vlan { 229 uint16_t id; 230 } __packed; 231 232 /* 233 * if_viovar.h: 234 */ 235 enum vio_ctrl_state { 236 FREE, INUSE, DONE, RESET 237 }; 238 239 struct vio_queue { 240 struct vio_softc *viq_sc; 241 struct virtio_net_hdr *viq_txhdrs; 242 bus_dmamap_t *viq_arrays; 243 #define viq_rxdmamaps viq_arrays 244 bus_dmamap_t *viq_txdmamaps; 245 struct mbuf **viq_rxmbufs; 246 struct mbuf **viq_txmbufs; 247 struct if_rxring viq_rxring; 248 struct ifiqueue *viq_ifiq; 249 struct ifqueue *viq_ifq; 250 struct virtqueue *viq_rxvq; 251 struct virtqueue *viq_txvq; 252 struct mutex viq_txmtx, viq_rxmtx; 253 int viq_txfree_slots; 254 } __aligned(CACHELINESIZE); 255 256 struct vio_softc { 257 struct device sc_dev; 258 259 struct virtio_softc *sc_virtio; 260 struct virtqueue *sc_ctl_vq; 261 262 struct arpcom sc_ac; 263 struct ifmedia sc_media; 264 265 short sc_ifflags; 266 267 /* bus_dmamem */ 268 bus_dma_segment_t sc_dma_seg; 269 bus_dmamap_t sc_dma_map; 270 size_t sc_dma_size; 271 caddr_t sc_dma_kva; 272 273 int sc_hdr_size; 274 struct virtio_net_ctrl_cmd *sc_ctrl_cmd; 275 struct virtio_net_ctrl_status *sc_ctrl_status; 276 struct virtio_net_ctrl_rx *sc_ctrl_rx; 277 struct virtio_net_ctrl_mq_pairs_set *sc_ctrl_mq_pairs; 278 struct virtio_net_ctrl_guest_offloads *sc_ctrl_guest_offloads; 279 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc; 280 #define sc_ctrl_mac_info sc_ctrl_mac_tbl_uc 281 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc; 282 283 struct intrmap *sc_intrmap; 284 struct vio_queue *sc_q; 285 uint16_t sc_nqueues; 286 int sc_tx_slots_per_req; 287 int sc_rx_mbuf_size; 288 289 enum vio_ctrl_state sc_ctrl_inuse; 290 291 struct timeout sc_txtick, sc_rxtick; 292 }; 293 294 #define VIO_DMAMEM_OFFSET(sc, p) ((caddr_t)(p) - (sc)->sc_dma_kva) 295 #define VIO_DMAMEM_SYNC(vsc, sc, p, size, flags) \ 296 bus_dmamap_sync((vsc)->sc_dmat, (sc)->sc_dma_map, \ 297 VIO_DMAMEM_OFFSET((sc), (p)), (size), (flags)) 298 #define VIO_HAVE_MRG_RXBUF(sc) \ 299 ((sc)->sc_hdr_size == sizeof(struct virtio_net_hdr)) 300 301 /* vioq N uses the rx/tx vq pair 2*N and 2*N + 1 */ 302 #define VIO_VQ2Q(sc, vq) (&sc->sc_q[vq->vq_index/2]) 303 304 #define VIRTIO_NET_CTRL_MAC_MC_ENTRIES 64 /* for more entries, use ALLMULTI */ 305 #define VIRTIO_NET_CTRL_MAC_UC_ENTRIES 1 /* one entry for own unicast addr */ 306 #define VIRTIO_NET_CTRL_TIMEOUT (5*1000*1000*1000ULL) /* 5 seconds */ 307 308 #define VIO_CTRL_MAC_INFO_SIZE \ 309 (2*sizeof(struct virtio_net_ctrl_mac_tbl) + \ 310 (VIRTIO_NET_CTRL_MAC_MC_ENTRIES + \ 311 VIRTIO_NET_CTRL_MAC_UC_ENTRIES) * ETHER_ADDR_LEN) 312 313 /* cfattach interface functions */ 314 int vio_match(struct device *, void *, void *); 315 void vio_attach(struct device *, struct device *, void *); 316 317 /* ifnet interface functions */ 318 int vio_init(struct ifnet *); 319 void vio_stop(struct ifnet *, int); 320 void vio_start(struct ifqueue *); 321 int vio_ioctl(struct ifnet *, u_long, caddr_t); 322 void vio_get_lladdr(struct arpcom *ac, struct virtio_softc *vsc); 323 void vio_put_lladdr(struct arpcom *ac, struct virtio_softc *vsc); 324 325 /* rx */ 326 int vio_add_rx_mbuf(struct vio_softc *, struct vio_queue *, int); 327 void vio_free_rx_mbuf(struct vio_softc *, struct vio_queue *, int); 328 void vio_populate_rx_mbufs(struct vio_softc *, struct vio_queue *); 329 int vio_rxeof(struct vio_queue *); 330 int vio_rx_intr(struct virtqueue *); 331 void vio_rx_drain(struct vio_softc *); 332 void vio_rxtick(void *); 333 334 /* tx */ 335 int vio_tx_intr(struct virtqueue *); 336 int vio_tx_dequeue(struct virtqueue *); 337 int vio_txeof(struct virtqueue *); 338 void vio_tx_drain(struct vio_softc *); 339 int vio_encap(struct vio_queue *, int, struct mbuf *); 340 void vio_txtick(void *); 341 342 int vio_queue_intr(void *); 343 int vio_config_intr(void *); 344 int vio_ctrl_intr(void *); 345 346 /* other control */ 347 void vio_link_state(struct ifnet *); 348 int vio_config_change(struct virtio_softc *); 349 int vio_ctrl_rx(struct vio_softc *, int, int); 350 int vio_ctrl_mq(struct vio_softc *); 351 int vio_ctrl_guest_offloads(struct vio_softc *, uint64_t); 352 int vio_set_rx_filter(struct vio_softc *); 353 void vio_iff(struct vio_softc *); 354 int vio_media_change(struct ifnet *); 355 void vio_media_status(struct ifnet *, struct ifmediareq *); 356 int vio_ctrleof(struct virtqueue *); 357 int vio_ctrl_start(struct vio_softc *, uint8_t, uint8_t, int, int *); 358 int vio_ctrl_submit(struct vio_softc *, int); 359 void vio_ctrl_finish(struct vio_softc *); 360 void vio_ctrl_wakeup(struct vio_softc *, enum vio_ctrl_state); 361 int vio_alloc_mem(struct vio_softc *, int); 362 int vio_alloc_dmamem(struct vio_softc *); 363 void vio_free_dmamem(struct vio_softc *); 364 365 #if VIRTIO_DEBUG 366 void vio_dump(struct vio_softc *); 367 #endif 368 369 int 370 vio_match(struct device *parent, void *match, void *aux) 371 { 372 struct virtio_attach_args *va = aux; 373 374 if (va->va_devid == PCI_PRODUCT_VIRTIO_NETWORK) 375 return 1; 376 377 return 0; 378 } 379 380 const struct cfattach vio_ca = { 381 sizeof(struct vio_softc), vio_match, vio_attach, NULL 382 }; 383 384 struct cfdriver vio_cd = { 385 NULL, "vio", DV_IFNET 386 }; 387 388 int 389 vio_alloc_dmamem(struct vio_softc *sc) 390 { 391 struct virtio_softc *vsc = sc->sc_virtio; 392 int nsegs; 393 394 if (bus_dmamap_create(vsc->sc_dmat, sc->sc_dma_size, 1, 395 sc->sc_dma_size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 396 &sc->sc_dma_map) != 0) 397 goto err; 398 if (bus_dmamem_alloc(vsc->sc_dmat, sc->sc_dma_size, 16, 0, 399 &sc->sc_dma_seg, 1, &nsegs, BUS_DMA_NOWAIT | BUS_DMA_ZERO) != 0) 400 goto destroy; 401 if (bus_dmamem_map(vsc->sc_dmat, &sc->sc_dma_seg, nsegs, 402 sc->sc_dma_size, &sc->sc_dma_kva, BUS_DMA_NOWAIT) != 0) 403 goto free; 404 if (bus_dmamap_load(vsc->sc_dmat, sc->sc_dma_map, sc->sc_dma_kva, 405 sc->sc_dma_size, NULL, BUS_DMA_NOWAIT) != 0) 406 goto unmap; 407 return (0); 408 409 unmap: 410 bus_dmamem_unmap(vsc->sc_dmat, sc->sc_dma_kva, sc->sc_dma_size); 411 free: 412 bus_dmamem_free(vsc->sc_dmat, &sc->sc_dma_seg, 1); 413 destroy: 414 bus_dmamap_destroy(vsc->sc_dmat, sc->sc_dma_map); 415 err: 416 return (1); 417 } 418 419 void 420 vio_free_dmamem(struct vio_softc *sc) 421 { 422 struct virtio_softc *vsc = sc->sc_virtio; 423 424 bus_dmamap_unload(vsc->sc_dmat, sc->sc_dma_map); 425 bus_dmamem_unmap(vsc->sc_dmat, sc->sc_dma_kva, sc->sc_dma_size); 426 bus_dmamem_free(vsc->sc_dmat, &sc->sc_dma_seg, 1); 427 bus_dmamap_destroy(vsc->sc_dmat, sc->sc_dma_map); 428 } 429 430 /* allocate memory */ 431 /* 432 * dma memory is used for: 433 * viq_txhdrs[slot]: metadata array for frames to be sent (WRITE) 434 * sc_ctrl_cmd: command to be sent via ctrl vq (WRITE) 435 * sc_ctrl_status: return value for a command via ctrl vq (READ) 436 * sc_ctrl_rx: parameter for a VIRTIO_NET_CTRL_RX class command 437 * (WRITE) 438 * sc_ctrl_mq_pairs_set: set number of rx/tx queue pais (WRITE) 439 * sc_ctrl_guest_offloads: configure offload features (WRITE) 440 * sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 441 * class command (WRITE) 442 * sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 443 * class command (WRITE) 444 * sc_ctrl_* structures are allocated only one each; they are protected by 445 * sc_ctrl_inuse, which must only be accessed at splnet 446 * 447 * metadata headers for received frames are stored at the start of the 448 * rx mbufs. 449 */ 450 /* 451 * dynamically allocated memory is used for: 452 * viq_rxdmamaps[slot]: bus_dmamap_t array for received payload 453 * viq_txdmamaps[slot]: bus_dmamap_t array for sent payload 454 * viq_rxmbufs[slot]: mbuf pointer array for received frames 455 * viq_txmbufs[slot]: mbuf pointer array for sent frames 456 */ 457 int 458 vio_alloc_mem(struct vio_softc *sc, int tx_max_segments) 459 { 460 struct virtio_softc *vsc = sc->sc_virtio; 461 struct ifnet *ifp = &sc->sc_ac.ac_if; 462 size_t allocsize, rxqsize, txqsize, offset = 0; 463 bus_size_t txsize; 464 caddr_t kva; 465 int i, qidx, r; 466 467 rxqsize = sc->sc_q[0].viq_rxvq->vq_num; 468 txqsize = sc->sc_q[0].viq_txvq->vq_num; 469 470 /* 471 * For simplicity, we always allocate the full virtio_net_hdr size 472 * even if VIRTIO_NET_F_MRG_RXBUF is not negotiated and 473 * only a part of the memory is ever used. 474 */ 475 allocsize = sizeof(struct virtio_net_hdr) * txqsize * sc->sc_nqueues; 476 477 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) { 478 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 479 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 480 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 481 allocsize += sizeof(struct virtio_net_ctrl_mq_pairs_set) * 1; 482 allocsize += sizeof(struct virtio_net_ctrl_guest_offloads) * 1; 483 allocsize += VIO_CTRL_MAC_INFO_SIZE; 484 } 485 sc->sc_dma_size = allocsize; 486 487 if (vio_alloc_dmamem(sc) != 0) { 488 printf("unable to allocate dma region\n"); 489 return -1; 490 } 491 492 kva = sc->sc_dma_kva; 493 494 for (qidx = 0; qidx < sc->sc_nqueues; qidx++) { 495 sc->sc_q[qidx].viq_txhdrs = 496 (struct virtio_net_hdr *)(kva + offset); 497 offset += sizeof(struct virtio_net_hdr) * txqsize; 498 } 499 500 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) { 501 sc->sc_ctrl_cmd = (void *)(kva + offset); 502 offset += sizeof(*sc->sc_ctrl_cmd); 503 sc->sc_ctrl_status = (void *)(kva + offset); 504 offset += sizeof(*sc->sc_ctrl_status); 505 sc->sc_ctrl_rx = (void *)(kva + offset); 506 offset += sizeof(*sc->sc_ctrl_rx); 507 sc->sc_ctrl_mq_pairs = (void *)(kva + offset); 508 offset += sizeof(*sc->sc_ctrl_mq_pairs); 509 sc->sc_ctrl_guest_offloads = (void *)(kva + offset); 510 offset += sizeof(*sc->sc_ctrl_guest_offloads); 511 sc->sc_ctrl_mac_tbl_uc = (void *)(kva + offset); 512 offset += sizeof(*sc->sc_ctrl_mac_tbl_uc) + 513 ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_UC_ENTRIES; 514 sc->sc_ctrl_mac_tbl_mc = (void *)(kva + offset); 515 offset += sizeof(*sc->sc_ctrl_mac_tbl_mc) + 516 ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MC_ENTRIES; 517 } 518 KASSERT(offset == allocsize); 519 520 if (virtio_has_feature(vsc, VIRTIO_NET_F_HOST_TSO4) || 521 virtio_has_feature(vsc, VIRTIO_NET_F_HOST_TSO6)) 522 txsize = MAXMCLBYTES + sc->sc_hdr_size + ETHER_HDR_LEN; 523 else 524 txsize = ifp->if_hardmtu + sc->sc_hdr_size + ETHER_HDR_LEN; 525 526 for (qidx = 0; qidx < sc->sc_nqueues; qidx++) { 527 struct vio_queue *vioq = &sc->sc_q[qidx]; 528 529 vioq->viq_arrays = mallocarray(rxqsize + txqsize, 530 sizeof(bus_dmamap_t) + sizeof(struct mbuf *), M_DEVBUF, 531 M_WAITOK|M_ZERO); 532 if (vioq->viq_arrays == NULL) { 533 printf("unable to allocate mem for dmamaps\n"); 534 goto free; 535 } 536 537 vioq->viq_txdmamaps = vioq->viq_arrays + rxqsize; 538 vioq->viq_rxmbufs = (void *)(vioq->viq_txdmamaps + txqsize); 539 vioq->viq_txmbufs = vioq->viq_rxmbufs + rxqsize; 540 541 for (i = 0; i < rxqsize; i++) { 542 r = bus_dmamap_create(vsc->sc_dmat, 543 sc->sc_rx_mbuf_size + sc->sc_hdr_size, 2, 544 sc->sc_rx_mbuf_size, 0, 545 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, 546 &vioq->viq_rxdmamaps[i]); 547 if (r != 0) 548 goto destroy; 549 } 550 551 for (i = 0; i < txqsize; i++) { 552 r = bus_dmamap_create(vsc->sc_dmat, txsize, 553 tx_max_segments, txsize, 0, 554 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, 555 &vioq->viq_txdmamaps[i]); 556 if (r != 0) 557 goto destroy; 558 } 559 } 560 561 return 0; 562 563 destroy: 564 printf("dmamap creation failed, error %d\n", r); 565 for (qidx = 0; qidx < sc->sc_nqueues; qidx++) { 566 struct vio_queue *vioq = &sc->sc_q[qidx]; 567 568 for (i = 0; i < txqsize; i++) { 569 if (vioq->viq_txdmamaps[i] == NULL) 570 break; 571 bus_dmamap_destroy(vsc->sc_dmat, 572 vioq->viq_txdmamaps[i]); 573 } 574 for (i = 0; i < rxqsize; i++) { 575 if (vioq->viq_rxdmamaps[i] == NULL) 576 break; 577 bus_dmamap_destroy(vsc->sc_dmat, 578 vioq->viq_rxdmamaps[i]); 579 } 580 free(vioq->viq_arrays, M_DEVBUF, (rxqsize + txqsize) * 581 (sizeof(bus_dmamap_t) + sizeof(struct mbuf *))); 582 vioq->viq_arrays = NULL; 583 } 584 free: 585 vio_free_dmamem(sc); 586 return -1; 587 } 588 589 static void 590 vio_dmamem_enqueue(struct virtio_softc *vsc, struct vio_softc *sc, 591 struct virtqueue *vq, int slot, void *p, size_t size, int write) 592 { 593 VIO_DMAMEM_SYNC(vsc, sc, p, size, write ? BUS_DMASYNC_PREWRITE : 594 BUS_DMASYNC_PREREAD); 595 virtio_enqueue_p(vq, slot, sc->sc_dma_map, VIO_DMAMEM_OFFSET(sc, p), 596 size, write); 597 } 598 599 void 600 vio_get_lladdr(struct arpcom *ac, struct virtio_softc *vsc) 601 { 602 int i; 603 for (i = 0; i < ETHER_ADDR_LEN; i++) { 604 ac->ac_enaddr[i] = virtio_read_device_config_1(vsc, 605 VIRTIO_NET_CONFIG_MAC + i); 606 } 607 } 608 609 void 610 vio_put_lladdr(struct arpcom *ac, struct virtio_softc *vsc) 611 { 612 int i; 613 for (i = 0; i < ETHER_ADDR_LEN; i++) { 614 virtio_write_device_config_1(vsc, VIRTIO_NET_CONFIG_MAC + i, 615 ac->ac_enaddr[i]); 616 } 617 } 618 619 static int 620 vio_needs_reset(struct vio_softc *sc) 621 { 622 if (virtio_get_status(sc->sc_virtio) & 623 VIRTIO_CONFIG_DEVICE_STATUS_DEVICE_NEEDS_RESET) { 624 printf("%s: device needs reset\n", sc->sc_dev.dv_xname); 625 vio_ctrl_wakeup(sc, RESET); 626 return 1; 627 } 628 return 0; 629 } 630 631 void 632 vio_attach(struct device *parent, struct device *self, void *aux) 633 { 634 struct vio_softc *sc = (struct vio_softc *)self; 635 struct virtio_softc *vsc = (struct virtio_softc *)parent; 636 struct virtio_attach_args *va = aux; 637 int i, r, tx_max_segments; 638 struct ifnet *ifp = &sc->sc_ac.ac_if; 639 640 if (vsc->sc_child != NULL) { 641 printf(": child already attached for %s; something wrong...\n", 642 parent->dv_xname); 643 return; 644 } 645 646 sc->sc_virtio = vsc; 647 648 vsc->sc_child = self; 649 vsc->sc_ipl = IPL_NET | IPL_MPSAFE; 650 vsc->sc_driver_features = VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | 651 VIRTIO_NET_F_CTRL_VQ | VIRTIO_NET_F_CTRL_RX | 652 VIRTIO_NET_F_MRG_RXBUF | VIRTIO_NET_F_CSUM | 653 VIRTIO_F_RING_EVENT_IDX | VIRTIO_NET_F_GUEST_CSUM; 654 655 if (va->va_nintr > 3 && ncpus > 1) 656 vsc->sc_driver_features |= VIRTIO_NET_F_MQ; 657 658 vsc->sc_driver_features |= VIRTIO_NET_F_HOST_TSO4; 659 vsc->sc_driver_features |= VIRTIO_NET_F_HOST_TSO6; 660 661 vsc->sc_driver_features |= VIRTIO_NET_F_CTRL_GUEST_OFFLOADS; 662 vsc->sc_driver_features |= VIRTIO_NET_F_GUEST_TSO4; 663 vsc->sc_driver_features |= VIRTIO_NET_F_GUEST_TSO6; 664 665 if (virtio_negotiate_features(vsc, virtio_net_feature_names) != 0) 666 goto err; 667 668 if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) { 669 i = virtio_read_device_config_2(vsc, 670 VIRTIO_NET_CONFIG_MAX_QUEUES); 671 vsc->sc_nvqs = 2 * i + 1; 672 i = MIN(i, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); 673 sc->sc_intrmap = intrmap_create(&sc->sc_dev, i, 674 va->va_nintr - 2, 0); 675 sc->sc_nqueues = intrmap_count(sc->sc_intrmap); 676 printf(": %u queue%s", sc->sc_nqueues, 677 sc->sc_nqueues > 1 ? "s" : ""); 678 } else { 679 sc->sc_nqueues = 1; 680 printf(": 1 queue"); 681 vsc->sc_nvqs = 2; 682 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) 683 vsc->sc_nvqs++; 684 } 685 686 vsc->sc_vqs = mallocarray(vsc->sc_nvqs, sizeof(*vsc->sc_vqs), M_DEVBUF, 687 M_WAITOK|M_ZERO); 688 if (vsc->sc_vqs == NULL) { 689 vsc->sc_nvqs = 0; 690 goto err; 691 } 692 693 sc->sc_q = mallocarray(sc->sc_nqueues, sizeof(*sc->sc_q), M_DEVBUF, 694 M_WAITOK|M_ZERO); 695 if (sc->sc_q == NULL) 696 goto err; 697 698 if (virtio_has_feature(vsc, VIRTIO_NET_F_MAC)) { 699 vio_get_lladdr(&sc->sc_ac, vsc); 700 } else { 701 ether_fakeaddr(ifp); 702 vio_put_lladdr(&sc->sc_ac, vsc); 703 } 704 printf(", address %s\n", ether_sprintf(sc->sc_ac.ac_enaddr)); 705 706 if (virtio_has_feature(vsc, VIRTIO_NET_F_MRG_RXBUF) || 707 vsc->sc_version_1) { 708 sc->sc_hdr_size = sizeof(struct virtio_net_hdr); 709 } else { 710 sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); 711 } 712 713 ifp->if_capabilities = 0; 714 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 715 ifp->if_xflags = IFXF_MPSAFE; 716 #if NVLAN > 0 717 ifp->if_capabilities |= IFCAP_VLAN_MTU; 718 ifp->if_capabilities |= IFCAP_VLAN_HWOFFLOAD; 719 #endif 720 if (virtio_has_feature(vsc, VIRTIO_NET_F_CSUM)) 721 ifp->if_capabilities |= IFCAP_CSUM_TCPv4|IFCAP_CSUM_UDPv4| 722 IFCAP_CSUM_TCPv6|IFCAP_CSUM_UDPv6; 723 if (virtio_has_feature(vsc, VIRTIO_NET_F_HOST_TSO4)) 724 ifp->if_capabilities |= IFCAP_TSOv4; 725 if (virtio_has_feature(vsc, VIRTIO_NET_F_HOST_TSO6)) 726 ifp->if_capabilities |= IFCAP_TSOv6; 727 728 sc->sc_rx_mbuf_size = MCLBYTES; 729 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) && 730 (virtio_has_feature(vsc, VIRTIO_NET_F_GUEST_TSO4) || 731 virtio_has_feature(vsc, VIRTIO_NET_F_GUEST_TSO6))) { 732 ifp->if_xflags |= IFXF_LRO; 733 ifp->if_capabilities |= IFCAP_LRO; 734 sc->sc_rx_mbuf_size = 4 * 1024; 735 } 736 737 if (virtio_has_feature(vsc, VIRTIO_NET_F_MRG_RXBUF)) 738 ifp->if_hardmtu = MAXMCLBYTES; 739 else 740 ifp->if_hardmtu = sc->sc_rx_mbuf_size - sc->sc_hdr_size - 741 ETHER_HDR_LEN; 742 743 /* defrag for longer mbuf chains */ 744 tx_max_segments = 16; 745 if (virtio_has_feature(vsc, VIRTIO_NET_F_HOST_TSO4) || 746 virtio_has_feature(vsc, VIRTIO_NET_F_HOST_TSO6)) { 747 /* 748 * With TSO, we may get 64K packets and want to be able to 749 * send longer chains without defragmenting 750 */ 751 tx_max_segments = 32; 752 } 753 754 if (virtio_has_feature(vsc, VIRTIO_F_RING_INDIRECT_DESC)) 755 sc->sc_tx_slots_per_req = 1; 756 else 757 sc->sc_tx_slots_per_req = tx_max_segments + 1; 758 759 for (i = 0; i < sc->sc_nqueues; i++) { 760 int vqidx = 2 * i; 761 struct vio_queue *vioq = &sc->sc_q[i]; 762 763 vioq->viq_rxvq = &vsc->sc_vqs[vqidx]; 764 mtx_init(&vioq->viq_txmtx, IPL_NET); 765 mtx_init(&vioq->viq_rxmtx, IPL_NET); 766 vioq->viq_sc = sc; 767 if (virtio_alloc_vq(vsc, vioq->viq_rxvq, vqidx, 2, "rx") != 0) 768 goto err; 769 vioq->viq_rxvq->vq_done = vio_rx_intr; 770 virtio_start_vq_intr(vsc, vioq->viq_rxvq); 771 772 vqidx++; 773 vioq->viq_txvq = &vsc->sc_vqs[vqidx]; 774 if (virtio_alloc_vq(vsc, vioq->viq_txvq, vqidx, 775 tx_max_segments + 1, "tx") != 0) { 776 goto err; 777 } 778 vioq->viq_txvq->vq_done = vio_tx_intr; 779 if (virtio_has_feature(vsc, VIRTIO_F_RING_EVENT_IDX)) 780 virtio_postpone_intr_far(vioq->viq_txvq); 781 else 782 virtio_stop_vq_intr(vsc, vioq->viq_txvq); 783 vioq->viq_txfree_slots = vioq->viq_txvq->vq_num - 1; 784 KASSERT(vioq->viq_txfree_slots > sc->sc_tx_slots_per_req); 785 if (vioq->viq_txvq->vq_num != sc->sc_q[0].viq_txvq->vq_num) { 786 printf("inequal tx queue size %d: %d != %d\n", i, 787 vioq->viq_txvq->vq_num, 788 sc->sc_q[0].viq_txvq->vq_num); 789 goto err; 790 } 791 DPRINTF("%d: q %p rx %p tx %p\n", i, vioq, vioq->viq_rxvq, 792 vioq->viq_txvq); 793 794 if (sc->sc_intrmap != NULL) { 795 vioq->viq_rxvq->vq_intr_vec = i + 2; 796 vioq->viq_txvq->vq_intr_vec = i + 2; 797 } 798 } 799 800 /* control queue */ 801 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) { 802 i = 2; 803 if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) { 804 i = 2 * virtio_read_device_config_2(vsc, 805 VIRTIO_NET_CONFIG_MAX_QUEUES); 806 } 807 sc->sc_ctl_vq = &vsc->sc_vqs[i]; 808 if (virtio_alloc_vq(vsc, sc->sc_ctl_vq, i, 1, "control") != 0) 809 goto err; 810 sc->sc_ctl_vq->vq_done = vio_ctrleof; 811 if (sc->sc_intrmap != NULL) 812 sc->sc_ctl_vq->vq_intr_vec = 1; 813 virtio_start_vq_intr(vsc, sc->sc_ctl_vq); 814 } 815 816 if (sc->sc_intrmap) { 817 r = virtio_intr_establish(vsc, va, 0, NULL, vio_config_intr, 818 vsc); 819 if (r != 0) { 820 printf("%s: cannot alloc config intr: %d\n", 821 sc->sc_dev.dv_xname, r); 822 goto err; 823 } 824 r = virtio_intr_establish(vsc, va, 1, NULL, vio_ctrl_intr, 825 sc->sc_ctl_vq); 826 if (r != 0) { 827 printf("%s: cannot alloc ctrl intr: %d\n", 828 sc->sc_dev.dv_xname, r); 829 goto err; 830 } 831 for (i = 0; i < sc->sc_nqueues; i++) { 832 struct cpu_info *ci = NULL; 833 ci = intrmap_cpu(sc->sc_intrmap, i); 834 r = virtio_intr_establish(vsc, va, i + 2, ci, 835 vio_queue_intr, &sc->sc_q[i]); 836 if (r != 0) { 837 printf("%s: cannot alloc q%d intr: %d\n", 838 sc->sc_dev.dv_xname, i, r); 839 goto err; 840 } 841 } 842 } 843 844 if (vio_alloc_mem(sc, tx_max_segments) < 0) 845 goto err; 846 847 strlcpy(ifp->if_xname, self->dv_xname, IFNAMSIZ); 848 ifp->if_softc = sc; 849 ifp->if_qstart = vio_start; 850 ifp->if_ioctl = vio_ioctl; 851 852 ifq_init_maxlen(&ifp->if_snd, vsc->sc_vqs[1].vq_num - 1); 853 ifmedia_init(&sc->sc_media, 0, vio_media_change, vio_media_status); 854 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 855 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 856 vsc->sc_config_change = vio_config_change; 857 timeout_set(&sc->sc_txtick, vio_txtick, sc); 858 timeout_set(&sc->sc_rxtick, vio_rxtick, sc); 859 860 if (virtio_attach_finish(vsc, va) != 0) 861 goto err; 862 863 if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) { 864 /* ctrl queue works only after DRIVER_OK */ 865 vio_ctrl_mq(sc); 866 } 867 868 if_attach(ifp); 869 ether_ifattach(ifp); 870 vio_link_state(ifp); 871 872 if_attach_queues(ifp, sc->sc_nqueues); 873 if_attach_iqueues(ifp, sc->sc_nqueues); 874 875 for (i = 0; i < sc->sc_nqueues; i++) { 876 ifp->if_ifqs[i]->ifq_softc = &sc->sc_q[i]; 877 sc->sc_q[i].viq_ifq = ifp->if_ifqs[i]; 878 sc->sc_q[i].viq_ifiq = ifp->if_iqs[i]; 879 } 880 881 return; 882 883 err: 884 for (i = 0; i < vsc->sc_nvqs; i++) 885 virtio_free_vq(vsc, &vsc->sc_vqs[i]); 886 free(vsc->sc_vqs, M_DEVBUF, vsc->sc_nvqs * sizeof(*vsc->sc_vqs)); 887 free(sc->sc_q, M_DEVBUF, sc->sc_nqueues * sizeof(*sc->sc_q)); 888 vsc->sc_nvqs = 0; 889 vsc->sc_child = VIRTIO_CHILD_ERROR; 890 return; 891 } 892 893 /* check link status */ 894 void 895 vio_link_state(struct ifnet *ifp) 896 { 897 struct vio_softc *sc = ifp->if_softc; 898 struct virtio_softc *vsc = sc->sc_virtio; 899 int link_state = LINK_STATE_FULL_DUPLEX; 900 901 if (virtio_has_feature(vsc, VIRTIO_NET_F_STATUS)) { 902 int status = virtio_read_device_config_2(vsc, 903 VIRTIO_NET_CONFIG_STATUS); 904 if (!(status & VIRTIO_NET_S_LINK_UP)) 905 link_state = LINK_STATE_DOWN; 906 } 907 if (ifp->if_link_state != link_state) { 908 ifp->if_link_state = link_state; 909 if_link_state_change(ifp); 910 } 911 } 912 913 /* interrupt handlers for multi-queue */ 914 int 915 vio_queue_intr(void *arg) 916 { 917 struct vio_queue *vioq = arg; 918 struct virtio_softc *vsc = vioq->viq_sc->sc_virtio; 919 int r; 920 r = virtio_check_vq(vsc, vioq->viq_txvq); 921 r |= virtio_check_vq(vsc, vioq->viq_rxvq); 922 return r; 923 } 924 925 int 926 vio_config_intr(void *arg) 927 { 928 struct virtio_softc *vsc = arg; 929 return vio_config_change(vsc); 930 } 931 932 int 933 vio_ctrl_intr(void *arg) 934 { 935 struct virtqueue *vq = arg; 936 return virtio_check_vq(vq->vq_owner, vq); 937 } 938 939 940 int 941 vio_config_change(struct virtio_softc *vsc) 942 { 943 struct vio_softc *sc = (struct vio_softc *)vsc->sc_child; 944 KERNEL_LOCK(); 945 vio_link_state(&sc->sc_ac.ac_if); 946 vio_needs_reset(sc); 947 KERNEL_UNLOCK(); 948 return 1; 949 } 950 951 int 952 vio_media_change(struct ifnet *ifp) 953 { 954 /* Ignore */ 955 return (0); 956 } 957 958 void 959 vio_media_status(struct ifnet *ifp, struct ifmediareq *imr) 960 { 961 imr->ifm_active = IFM_ETHER | IFM_AUTO; 962 imr->ifm_status = IFM_AVALID; 963 964 vio_link_state(ifp); 965 if (LINK_STATE_IS_UP(ifp->if_link_state) && ifp->if_flags & IFF_UP) 966 imr->ifm_status |= IFM_ACTIVE|IFM_FDX; 967 } 968 969 /* 970 * Interface functions for ifnet 971 */ 972 int 973 vio_init(struct ifnet *ifp) 974 { 975 struct vio_softc *sc = ifp->if_softc; 976 struct virtio_softc *vsc = sc->sc_virtio; 977 int qidx; 978 979 vio_stop(ifp, 0); 980 for (qidx = 0; qidx < sc->sc_nqueues; qidx++) { 981 struct vio_queue *vioq = &sc->sc_q[qidx]; 982 983 mtx_enter(&vioq->viq_rxmtx); 984 if_rxr_init(&vioq->viq_rxring, 985 2 * ((ifp->if_hardmtu / sc->sc_rx_mbuf_size) + 1), 986 vioq->viq_rxvq->vq_num); 987 vio_populate_rx_mbufs(sc, vioq); 988 ifq_clr_oactive(vioq->viq_ifq); 989 mtx_leave(&vioq->viq_rxmtx); 990 } 991 vio_iff(sc); 992 vio_link_state(ifp); 993 994 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 995 uint64_t features = 0; 996 997 if (virtio_has_feature(vsc, VIRTIO_NET_F_GUEST_CSUM)) 998 SET(features, VIRTIO_NET_F_GUEST_CSUM); 999 1000 if (ISSET(ifp->if_xflags, IFXF_LRO)) { 1001 if (virtio_has_feature(vsc, VIRTIO_NET_F_GUEST_TSO4)) 1002 SET(features, VIRTIO_NET_F_GUEST_TSO4); 1003 if (virtio_has_feature(vsc, VIRTIO_NET_F_GUEST_TSO6)) 1004 SET(features, VIRTIO_NET_F_GUEST_TSO6); 1005 } 1006 1007 vio_ctrl_guest_offloads(sc, features); 1008 } 1009 1010 SET(ifp->if_flags, IFF_RUNNING); 1011 1012 return 0; 1013 } 1014 1015 void 1016 vio_stop(struct ifnet *ifp, int disable) 1017 { 1018 struct vio_softc *sc = ifp->if_softc; 1019 struct virtio_softc *vsc = sc->sc_virtio; 1020 int i; 1021 1022 CLR(ifp->if_flags, IFF_RUNNING); 1023 timeout_del(&sc->sc_txtick); 1024 timeout_del(&sc->sc_rxtick); 1025 /* only way to stop I/O and DMA is resetting... */ 1026 virtio_reset(vsc); 1027 virtio_intr_barrier(vsc); 1028 for (i = 0; i < sc->sc_nqueues; i++) { 1029 mtx_enter(&sc->sc_q[i].viq_rxmtx); 1030 vio_rxeof(&sc->sc_q[i]); 1031 mtx_leave(&sc->sc_q[i].viq_rxmtx); 1032 } 1033 1034 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) 1035 vio_ctrl_wakeup(sc, RESET); 1036 vio_tx_drain(sc); 1037 if (disable) 1038 vio_rx_drain(sc); 1039 1040 virtio_reinit_start(vsc); 1041 for (i = 0; i < sc->sc_nqueues; i++) { 1042 virtio_start_vq_intr(vsc, sc->sc_q[i].viq_rxvq); 1043 virtio_stop_vq_intr(vsc, sc->sc_q[i].viq_txvq); 1044 } 1045 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) 1046 virtio_start_vq_intr(vsc, sc->sc_ctl_vq); 1047 virtio_reinit_end(vsc); 1048 if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) 1049 vio_ctrl_mq(sc); 1050 if (virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_VQ)) 1051 vio_ctrl_wakeup(sc, FREE); 1052 } 1053 1054 static inline uint16_t 1055 vio_cksum_update(uint32_t cksum, uint16_t paylen) 1056 { 1057 /* Add payload length */ 1058 cksum += paylen; 1059 1060 /* Fold back to 16 bit */ 1061 cksum += cksum >> 16; 1062 1063 return (uint16_t)(cksum); 1064 } 1065 1066 void 1067 vio_tx_offload(struct virtio_net_hdr *hdr, struct mbuf *m) 1068 { 1069 struct ether_extracted ext; 1070 1071 /* 1072 * Checksum Offload 1073 */ 1074 1075 if (!ISSET(m->m_pkthdr.csum_flags, M_TCP_CSUM_OUT) && 1076 !ISSET(m->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) 1077 return; 1078 1079 ether_extract_headers(m, &ext); 1080 1081 /* Consistency Checks */ 1082 if ((!ext.ip4 && !ext.ip6) || (!ext.tcp && !ext.udp)) 1083 return; 1084 1085 if ((ext.tcp && !ISSET(m->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) || 1086 (ext.udp && !ISSET(m->m_pkthdr.csum_flags, M_UDP_CSUM_OUT))) 1087 return; 1088 1089 hdr->csum_start = sizeof(*ext.eh); 1090 #if NVLAN > 0 1091 if (ext.evh) 1092 hdr->csum_start = sizeof(*ext.evh); 1093 #endif 1094 hdr->csum_start += ext.iphlen; 1095 1096 if (ext.tcp) 1097 hdr->csum_offset = offsetof(struct tcphdr, th_sum); 1098 else if (ext.udp) 1099 hdr->csum_offset = offsetof(struct udphdr, uh_sum); 1100 1101 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 1102 1103 /* 1104 * TCP Segmentation Offload 1105 */ 1106 1107 if (!ISSET(m->m_pkthdr.csum_flags, M_TCP_TSO)) 1108 return; 1109 1110 if (!ext.tcp || m->m_pkthdr.ph_mss == 0) { 1111 tcpstat_inc(tcps_outbadtso); 1112 return; 1113 } 1114 1115 hdr->hdr_len = hdr->csum_start + ext.tcphlen; 1116 hdr->gso_size = m->m_pkthdr.ph_mss; 1117 1118 if (ext.ip4) 1119 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 1120 #ifdef INET6 1121 else if (ext.ip6) 1122 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 1123 #endif 1124 1125 /* 1126 * VirtIO-Net needs pseudo header cksum with IP-payload length for TSO 1127 */ 1128 ext.tcp->th_sum = vio_cksum_update(ext.tcp->th_sum, 1129 htons(ext.iplen - ext.iphlen)); 1130 1131 tcpstat_add(tcps_outpkttso, 1132 (ext.paylen + m->m_pkthdr.ph_mss - 1) / m->m_pkthdr.ph_mss); 1133 } 1134 1135 void 1136 vio_start(struct ifqueue *viq_ifq) 1137 { 1138 struct ifnet *ifp = viq_ifq->ifq_if; 1139 struct vio_queue *vioq = viq_ifq->ifq_softc; 1140 struct vio_softc *sc = ifp->if_softc; 1141 struct virtio_softc *vsc = sc->sc_virtio; 1142 struct virtqueue *vq = vioq->viq_txvq; 1143 struct mbuf *m; 1144 int queued = 0, free_slots, used_slots, r; 1145 1146 mtx_enter(&vioq->viq_txmtx); 1147 r = vio_tx_dequeue(vq); 1148 if (r && ifq_is_oactive(viq_ifq)) 1149 ifq_clr_oactive(viq_ifq); 1150 1151 again: 1152 free_slots = vioq->viq_txfree_slots; 1153 KASSERT(free_slots >= 0); 1154 used_slots = 0; 1155 for (;;) { 1156 int slot; 1157 struct virtio_net_hdr *hdr; 1158 1159 if (free_slots - used_slots < sc->sc_tx_slots_per_req) { 1160 ifq_set_oactive(viq_ifq); 1161 break; 1162 } 1163 1164 m = ifq_dequeue(viq_ifq); 1165 if (m == NULL) 1166 break; 1167 1168 r = virtio_enqueue_prep(vq, &slot); 1169 if (r == EAGAIN) { 1170 printf("%s: virtio_enqueue_prep failed?\n", __func__); 1171 m_freem(m); 1172 viq_ifq->ifq_errors++; 1173 break; 1174 } 1175 if (r != 0) 1176 panic("%s: enqueue_prep for tx buffer: %d", 1177 sc->sc_dev.dv_xname, r); 1178 1179 hdr = &vioq->viq_txhdrs[slot]; 1180 memset(hdr, 0, sc->sc_hdr_size); 1181 vio_tx_offload(hdr, m); 1182 1183 r = vio_encap(vioq, slot, m); 1184 if (r != 0) { 1185 virtio_enqueue_abort(vq, slot); 1186 m_freem(m); 1187 viq_ifq->ifq_errors++; 1188 continue; 1189 } 1190 r = virtio_enqueue_reserve(vq, slot, 1191 vioq->viq_txdmamaps[slot]->dm_nsegs + 1); 1192 if (r != 0) { 1193 printf("%s: virtio_enqueue_reserve failed?\n", 1194 __func__); 1195 m_freem(m); 1196 viq_ifq->ifq_errors++; 1197 bus_dmamap_unload(vsc->sc_dmat, 1198 vioq->viq_txdmamaps[slot]); 1199 vioq->viq_txmbufs[slot] = NULL; 1200 break; 1201 } 1202 if (sc->sc_tx_slots_per_req == 1) 1203 used_slots++; 1204 else 1205 used_slots += vioq->viq_txdmamaps[slot]->dm_nsegs + 1; 1206 1207 1208 bus_dmamap_sync(vsc->sc_dmat, vioq->viq_txdmamaps[slot], 0, 1209 vioq->viq_txdmamaps[slot]->dm_mapsize, 1210 BUS_DMASYNC_PREWRITE); 1211 vio_dmamem_enqueue(vsc, sc, vq, slot, hdr, sc->sc_hdr_size, 1); 1212 virtio_enqueue(vq, slot, vioq->viq_txdmamaps[slot], 1); 1213 virtio_enqueue_commit(vsc, vq, slot, 0); 1214 queued++; 1215 #if NBPFILTER > 0 1216 if (ifp->if_bpf) 1217 bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT); 1218 #endif 1219 } 1220 if (used_slots > 0) { 1221 if (used_slots > vioq->viq_txfree_slots) 1222 printf("%s: used_slots %d viq_txfree_slots %d " 1223 "free_slots %d\n", __func__, used_slots, 1224 vioq->viq_txfree_slots, free_slots); 1225 vioq->viq_txfree_slots -= used_slots; 1226 KASSERT(vioq->viq_txfree_slots >= 0); 1227 } 1228 if (ifq_is_oactive(viq_ifq) && ISSET(ifp->if_flags, IFF_RUNNING)) { 1229 if (virtio_has_feature(vsc, VIRTIO_F_RING_EVENT_IDX)) 1230 r = virtio_postpone_intr_smart(vq); 1231 else 1232 r = virtio_start_vq_intr(vsc, vq); 1233 if (r) { 1234 r = vio_tx_dequeue(vq); 1235 if (r) 1236 ifq_clr_oactive(viq_ifq); 1237 goto again; 1238 } 1239 } 1240 1241 if (queued > 0) { 1242 virtio_notify(vsc, vq); 1243 timeout_add_sec(&sc->sc_txtick, 1); 1244 } 1245 mtx_leave(&vioq->viq_txmtx); 1246 } 1247 1248 #if VIRTIO_DEBUG 1249 void 1250 vio_dump(struct vio_softc *sc) 1251 { 1252 struct ifnet *ifp = &sc->sc_ac.ac_if; 1253 int i; 1254 1255 printf("%s status dump:\n", ifp->if_xname); 1256 printf("tx tick active: %d\n", !timeout_triggered(&sc->sc_txtick)); 1257 printf("max tx slots per req %d\n", sc->sc_tx_slots_per_req); 1258 printf("rx tick active: %d\n", !timeout_triggered(&sc->sc_rxtick)); 1259 for (i = 0; i < sc->sc_nqueues; i++) { 1260 printf("%d: TX virtqueue:\n", i); 1261 printf(" tx free slots %d\n", sc->sc_q[i].viq_txfree_slots); 1262 virtio_vq_dump(sc->sc_q[i].viq_txvq); 1263 printf("%d: RX virtqueue:\n", i); 1264 virtio_vq_dump(sc->sc_q[i].viq_rxvq); 1265 } 1266 if (sc->sc_ctl_vq != NULL) { 1267 printf("CTL virtqueue:\n"); 1268 virtio_vq_dump(sc->sc_ctl_vq); 1269 printf("ctrl_inuse: %d\n", sc->sc_ctrl_inuse); 1270 } 1271 } 1272 #endif 1273 1274 static int 1275 vio_rxr_info(struct vio_softc *sc, struct if_rxrinfo *ifri) 1276 { 1277 struct if_rxring_info *ifrs, *ifr; 1278 int error; 1279 unsigned int i; 1280 1281 ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), 1282 M_TEMP, M_WAITOK|M_ZERO|M_CANFAIL); 1283 if (ifrs == NULL) 1284 return (ENOMEM); 1285 1286 for (i = 0; i < sc->sc_nqueues; i++) { 1287 ifr = &ifrs[i]; 1288 1289 ifr->ifr_size = sc->sc_rx_mbuf_size; 1290 snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i); 1291 ifr->ifr_info = sc->sc_q[i].viq_rxring; 1292 } 1293 1294 error = if_rxr_info_ioctl(ifri, i, ifrs); 1295 1296 free(ifrs, M_TEMP, i * sizeof(*ifrs)); 1297 1298 return (error); 1299 } 1300 1301 int 1302 vio_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 1303 { 1304 struct vio_softc *sc = ifp->if_softc; 1305 struct ifreq *ifr = (struct ifreq *)data; 1306 int s, r = 0; 1307 1308 s = splnet(); 1309 switch (cmd) { 1310 case SIOCSIFADDR: 1311 ifp->if_flags |= IFF_UP; 1312 if (!(ifp->if_flags & IFF_RUNNING)) 1313 vio_init(ifp); 1314 break; 1315 case SIOCSIFFLAGS: 1316 if (ifp->if_flags & IFF_UP) { 1317 #if VIRTIO_DEBUG 1318 if (ifp->if_flags & IFF_DEBUG) 1319 vio_dump(sc); 1320 #endif 1321 if (ifp->if_flags & IFF_RUNNING) 1322 r = ENETRESET; 1323 else 1324 vio_init(ifp); 1325 } else { 1326 if (ifp->if_flags & IFF_RUNNING) 1327 vio_stop(ifp, 1); 1328 } 1329 break; 1330 case SIOCGIFMEDIA: 1331 case SIOCSIFMEDIA: 1332 r = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 1333 break; 1334 case SIOCGIFRXR: 1335 r = vio_rxr_info(sc, (struct if_rxrinfo *)ifr->ifr_data); 1336 break; 1337 default: 1338 r = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 1339 } 1340 1341 if (r == ENETRESET) { 1342 if (ifp->if_flags & IFF_RUNNING) 1343 vio_iff(sc); 1344 r = 0; 1345 } 1346 splx(s); 1347 return r; 1348 } 1349 1350 /* 1351 * Receive implementation 1352 */ 1353 /* allocate and initialize a mbuf for receive */ 1354 int 1355 vio_add_rx_mbuf(struct vio_softc *sc, struct vio_queue *vioq, int i) 1356 { 1357 struct mbuf *m; 1358 int r; 1359 1360 m = MCLGETL(NULL, M_DONTWAIT, sc->sc_rx_mbuf_size); 1361 if (m == NULL) 1362 return ENOBUFS; 1363 vioq->viq_rxmbufs[i] = m; 1364 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 1365 /* XXX m_adj ETHER_ALIGN ? */ 1366 r = bus_dmamap_load_mbuf(sc->sc_virtio->sc_dmat, 1367 vioq->viq_rxdmamaps[i], m, BUS_DMA_READ|BUS_DMA_NOWAIT); 1368 if (r) { 1369 m_freem(m); 1370 vioq->viq_rxmbufs[i] = NULL; 1371 return r; 1372 } 1373 1374 return 0; 1375 } 1376 1377 /* free a mbuf for receive */ 1378 void 1379 vio_free_rx_mbuf(struct vio_softc *sc, struct vio_queue *vioq, int i) 1380 { 1381 bus_dmamap_unload(sc->sc_virtio->sc_dmat, vioq->viq_rxdmamaps[i]); 1382 m_freem(vioq->viq_rxmbufs[i]); 1383 vioq->viq_rxmbufs[i] = NULL; 1384 } 1385 1386 /* add mbufs for all the empty receive slots */ 1387 void 1388 vio_populate_rx_mbufs(struct vio_softc *sc, struct vio_queue *vioq) 1389 { 1390 struct virtio_softc *vsc = sc->sc_virtio; 1391 int r, done = 0; 1392 u_int slots; 1393 struct virtqueue *vq = vioq->viq_rxvq; 1394 int mrg_rxbuf = VIO_HAVE_MRG_RXBUF(sc); 1395 1396 MUTEX_ASSERT_LOCKED(&vioq->viq_rxmtx); 1397 for (slots = if_rxr_get(&vioq->viq_rxring, vq->vq_num); 1398 slots > 0; slots--) { 1399 int slot; 1400 r = virtio_enqueue_prep(vq, &slot); 1401 if (r == EAGAIN) 1402 break; 1403 if (r != 0) 1404 panic("%s: enqueue_prep for rx buffer: %d", 1405 sc->sc_dev.dv_xname, r); 1406 if (vioq->viq_rxmbufs[slot] == NULL) { 1407 r = vio_add_rx_mbuf(sc, vioq, slot); 1408 if (r != 0) { 1409 virtio_enqueue_abort(vq, slot); 1410 break; 1411 } 1412 } 1413 r = virtio_enqueue_reserve(vq, slot, 1414 vioq->viq_rxdmamaps[slot]->dm_nsegs + (mrg_rxbuf ? 0 : 1)); 1415 if (r != 0) { 1416 vio_free_rx_mbuf(sc, vioq, slot); 1417 break; 1418 } 1419 bus_dmamap_sync(vsc->sc_dmat, vioq->viq_rxdmamaps[slot], 0, 1420 vioq->viq_rxdmamaps[slot]->dm_mapsize, 1421 BUS_DMASYNC_PREREAD); 1422 if (mrg_rxbuf) { 1423 virtio_enqueue(vq, slot, vioq->viq_rxdmamaps[slot], 0); 1424 } else { 1425 /* 1426 * Buggy kvm wants a buffer of exactly the size of 1427 * the header in this case, so we have to split in 1428 * two. 1429 */ 1430 virtio_enqueue_p(vq, slot, vioq->viq_rxdmamaps[slot], 1431 0, sc->sc_hdr_size, 0); 1432 virtio_enqueue_p(vq, slot, vioq->viq_rxdmamaps[slot], 1433 sc->sc_hdr_size, 1434 sc->sc_rx_mbuf_size - sc->sc_hdr_size, 0); 1435 } 1436 virtio_enqueue_commit(vsc, vq, slot, 0); 1437 done = 1; 1438 } 1439 if_rxr_put(&vioq->viq_rxring, slots); 1440 1441 if (done) 1442 virtio_notify(vsc, vq); 1443 timeout_add_sec(&sc->sc_rxtick, 1); 1444 } 1445 1446 void 1447 vio_rx_offload(struct mbuf *m, struct virtio_net_hdr *hdr) 1448 { 1449 struct ether_extracted ext; 1450 1451 if (!ISSET(hdr->flags, VIRTIO_NET_HDR_F_DATA_VALID) && 1452 !ISSET(hdr->flags, VIRTIO_NET_HDR_F_NEEDS_CSUM)) 1453 return; 1454 1455 ether_extract_headers(m, &ext); 1456 1457 if (ext.tcp) { 1458 SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_OK); 1459 if (ISSET(hdr->flags, VIRTIO_NET_HDR_F_NEEDS_CSUM)) 1460 SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_OUT); 1461 } else if (ext.udp) { 1462 SET(m->m_pkthdr.csum_flags, M_UDP_CSUM_IN_OK); 1463 if (ISSET(hdr->flags, VIRTIO_NET_HDR_F_NEEDS_CSUM)) 1464 SET(m->m_pkthdr.csum_flags, M_UDP_CSUM_OUT); 1465 } 1466 1467 if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 1468 hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 1469 uint16_t mss = hdr->gso_size; 1470 1471 if (!ext.tcp || mss == 0) { 1472 tcpstat_inc(tcps_inbadlro); 1473 return; 1474 } 1475 1476 if ((ext.paylen + mss - 1) / mss <= 1) 1477 return; 1478 1479 tcpstat_inc(tcps_inhwlro); 1480 tcpstat_add(tcps_inpktlro, (ext.paylen + mss - 1) / mss); 1481 SET(m->m_pkthdr.csum_flags, M_TCP_TSO); 1482 m->m_pkthdr.ph_mss = mss; 1483 } 1484 } 1485 1486 /* dequeue received packets */ 1487 int 1488 vio_rxeof(struct vio_queue *vioq) 1489 { 1490 struct vio_softc *sc = vioq->viq_sc; 1491 struct virtio_softc *vsc = sc->sc_virtio; 1492 struct ifnet *ifp = &sc->sc_ac.ac_if; 1493 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 1494 struct mbuf *m, *m0 = NULL, *mlast; 1495 int r = 0; 1496 int slot, len, bufs_left; 1497 struct virtio_net_hdr *hdr; 1498 1499 MUTEX_ASSERT_LOCKED(&vioq->viq_rxmtx); 1500 while (virtio_dequeue(vsc, vioq->viq_rxvq, &slot, &len) == 0) { 1501 r = 1; 1502 bus_dmamap_sync(vsc->sc_dmat, vioq->viq_rxdmamaps[slot], 0, 1503 vioq->viq_rxdmamaps[slot]->dm_mapsize, 1504 BUS_DMASYNC_POSTREAD); 1505 m = vioq->viq_rxmbufs[slot]; 1506 KASSERT(m != NULL); 1507 bus_dmamap_unload(vsc->sc_dmat, vioq->viq_rxdmamaps[slot]); 1508 vioq->viq_rxmbufs[slot] = NULL; 1509 virtio_dequeue_commit(vioq->viq_rxvq, slot); 1510 if_rxr_put(&vioq->viq_rxring, 1); 1511 m->m_len = m->m_pkthdr.len = len; 1512 m->m_pkthdr.csum_flags = 0; 1513 if (m0 == NULL) { 1514 hdr = mtod(m, struct virtio_net_hdr *); 1515 m_adj(m, sc->sc_hdr_size); 1516 m0 = mlast = m; 1517 if (virtio_has_feature(vsc, VIRTIO_NET_F_MQ)) { 1518 m->m_pkthdr.ph_flowid = 1519 vioq->viq_ifiq->ifiq_idx; 1520 SET(m->m_pkthdr.csum_flags, M_FLOWID); 1521 } 1522 if (VIO_HAVE_MRG_RXBUF(sc)) 1523 bufs_left = hdr->num_buffers - 1; 1524 else 1525 bufs_left = 0; 1526 } else { 1527 m->m_flags &= ~M_PKTHDR; 1528 m0->m_pkthdr.len += m->m_len; 1529 mlast->m_next = m; 1530 mlast = m; 1531 bufs_left--; 1532 } 1533 1534 if (bufs_left == 0) { 1535 if (virtio_has_feature(vsc, VIRTIO_NET_F_GUEST_CSUM)) 1536 vio_rx_offload(m0, hdr); 1537 ml_enqueue(&ml, m0); 1538 m0 = NULL; 1539 } 1540 } 1541 if (m0 != NULL) { 1542 DPRINTF("%s: expected %u buffers, got %u\n", __func__, 1543 hdr->num_buffers, hdr->num_buffers - bufs_left); 1544 ifp->if_ierrors++; 1545 m_freem(m0); 1546 } 1547 1548 if (ifiq_input(vioq->viq_ifiq, &ml)) 1549 if_rxr_livelocked(&vioq->viq_rxring); 1550 1551 return r; 1552 } 1553 1554 int 1555 vio_rx_intr(struct virtqueue *vq) 1556 { 1557 struct virtio_softc *vsc = vq->vq_owner; 1558 struct vio_softc *sc = (struct vio_softc *)vsc->sc_child; 1559 struct vio_queue *vioq = VIO_VQ2Q(sc, vq); 1560 int r, sum = 0; 1561 1562 mtx_enter(&vioq->viq_rxmtx); 1563 again: 1564 r = vio_rxeof(vioq); 1565 sum += r; 1566 if (r) { 1567 vio_populate_rx_mbufs(sc, vioq); 1568 /* set used event index to the next slot */ 1569 if (virtio_has_feature(vsc, VIRTIO_F_RING_EVENT_IDX)) { 1570 if (virtio_start_vq_intr(vq->vq_owner, vq)) 1571 goto again; 1572 } 1573 } 1574 1575 mtx_leave(&vioq->viq_rxmtx); 1576 return sum; 1577 } 1578 1579 void 1580 vio_rxtick(void *arg) 1581 { 1582 struct vio_softc *sc = arg; 1583 int i; 1584 1585 for (i = 0; i < sc->sc_nqueues; i++) { 1586 mtx_enter(&sc->sc_q[i].viq_rxmtx); 1587 vio_populate_rx_mbufs(sc, &sc->sc_q[i]); 1588 mtx_leave(&sc->sc_q[i].viq_rxmtx); 1589 } 1590 } 1591 1592 /* free all the mbufs; called from if_stop(disable) */ 1593 void 1594 vio_rx_drain(struct vio_softc *sc) 1595 { 1596 struct vio_queue *vioq; 1597 int i, qidx; 1598 1599 for (qidx = 0; qidx < sc->sc_nqueues; qidx++) { 1600 vioq = &sc->sc_q[qidx]; 1601 for (i = 0; i < vioq->viq_rxvq->vq_num; i++) { 1602 if (vioq->viq_rxmbufs[i] == NULL) 1603 continue; 1604 vio_free_rx_mbuf(sc, vioq, i); 1605 } 1606 } 1607 } 1608 1609 /* 1610 * Transmission implementation 1611 */ 1612 /* actual transmission is done in if_start */ 1613 /* tx interrupt; dequeue and free mbufs */ 1614 /* 1615 * tx interrupt is actually disabled unless the tx queue is full, i.e. 1616 * IFF_OACTIVE is set. vio_txtick is used to make sure that mbufs 1617 * are dequeued and freed even if no further transfer happens. 1618 */ 1619 int 1620 vio_tx_intr(struct virtqueue *vq) 1621 { 1622 struct virtio_softc *vsc = vq->vq_owner; 1623 struct vio_softc *sc = (struct vio_softc *)vsc->sc_child; 1624 struct vio_queue *vioq = VIO_VQ2Q(sc, vq); 1625 int r; 1626 1627 r = vio_txeof(vq); 1628 vio_start(vioq->viq_ifq); 1629 return r; 1630 } 1631 1632 void 1633 vio_txtick(void *arg) 1634 { 1635 struct vio_softc *sc = arg; 1636 int i; 1637 1638 for (i = 0; i < sc->sc_nqueues; i++) 1639 virtio_check_vq(sc->sc_virtio, sc->sc_q[i].viq_txvq); 1640 } 1641 1642 int 1643 vio_tx_dequeue(struct virtqueue *vq) 1644 { 1645 struct virtio_softc *vsc = vq->vq_owner; 1646 struct vio_softc *sc = (struct vio_softc *)vsc->sc_child; 1647 struct vio_queue *vioq = VIO_VQ2Q(sc, vq); 1648 struct mbuf *m; 1649 int r = 0; 1650 int slot, len, freed = 0; 1651 1652 MUTEX_ASSERT_LOCKED(&vioq->viq_txmtx); 1653 1654 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1655 struct virtio_net_hdr *hdr = &vioq->viq_txhdrs[slot]; 1656 r++; 1657 VIO_DMAMEM_SYNC(vsc, sc, hdr, sc->sc_hdr_size, 1658 BUS_DMASYNC_POSTWRITE); 1659 bus_dmamap_sync(vsc->sc_dmat, vioq->viq_txdmamaps[slot], 0, 1660 vioq->viq_txdmamaps[slot]->dm_mapsize, 1661 BUS_DMASYNC_POSTWRITE); 1662 m = vioq->viq_txmbufs[slot]; 1663 bus_dmamap_unload(vsc->sc_dmat, vioq->viq_txdmamaps[slot]); 1664 vioq->viq_txmbufs[slot] = NULL; 1665 freed += virtio_dequeue_commit(vq, slot); 1666 m_freem(m); 1667 } 1668 KASSERT(vioq->viq_txfree_slots >= 0); 1669 vioq->viq_txfree_slots += freed; 1670 return r; 1671 } 1672 1673 1674 int 1675 vio_txeof(struct virtqueue *vq) 1676 { 1677 struct virtio_softc *vsc = vq->vq_owner; 1678 struct vio_softc *sc = (struct vio_softc *)vsc->sc_child; 1679 struct vio_queue *vioq = VIO_VQ2Q(sc, vq); 1680 int r; 1681 1682 mtx_enter(&vioq->viq_txmtx); 1683 r = vio_tx_dequeue(vq); 1684 mtx_leave(&vioq->viq_txmtx); 1685 1686 if (r) { 1687 if (ifq_is_oactive(vioq->viq_ifq)) { 1688 mtx_enter(&vioq->viq_txmtx); 1689 virtio_stop_vq_intr(vsc, vq); 1690 mtx_leave(&vioq->viq_txmtx); 1691 ifq_restart(vioq->viq_ifq); 1692 } 1693 } 1694 if (vq->vq_used_idx == vq->vq_avail_idx) 1695 timeout_del(&sc->sc_txtick); 1696 else if (r) 1697 timeout_add_sec(&sc->sc_txtick, 1); 1698 return r; 1699 } 1700 1701 int 1702 vio_encap(struct vio_queue *vioq, int slot, struct mbuf *m) 1703 { 1704 struct virtio_softc *vsc = vioq->viq_sc->sc_virtio; 1705 bus_dmamap_t dmap = vioq->viq_txdmamaps[slot]; 1706 int r; 1707 1708 r = bus_dmamap_load_mbuf(vsc->sc_dmat, dmap, m, 1709 BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1710 switch (r) { 1711 case 0: 1712 break; 1713 case EFBIG: 1714 if (m_defrag(m, M_DONTWAIT) == 0 && 1715 bus_dmamap_load_mbuf(vsc->sc_dmat, dmap, m, 1716 BUS_DMA_WRITE|BUS_DMA_NOWAIT) == 0) 1717 break; 1718 1719 /* FALLTHROUGH */ 1720 default: 1721 return ENOBUFS; 1722 } 1723 vioq->viq_txmbufs[slot] = m; 1724 return 0; 1725 } 1726 1727 /* free all the mbufs already put on vq; called from if_stop(disable) */ 1728 void 1729 vio_tx_drain(struct vio_softc *sc) 1730 { 1731 struct virtio_softc *vsc = sc->sc_virtio; 1732 struct vio_queue *vioq; 1733 int i, q; 1734 1735 for (q = 0; q < sc->sc_nqueues; q++) { 1736 vioq = &sc->sc_q[q]; 1737 ifq_barrier(vioq->viq_ifq); 1738 mtx_enter(&vioq->viq_txmtx); 1739 for (i = 0; i < vioq->viq_txvq->vq_num; i++) { 1740 if (vioq->viq_txmbufs[i] == NULL) 1741 continue; 1742 bus_dmamap_unload(vsc->sc_dmat, 1743 vioq->viq_txdmamaps[i]); 1744 m_freem(vioq->viq_txmbufs[i]); 1745 vioq->viq_txmbufs[i] = NULL; 1746 } 1747 ifq_purge(vioq->viq_ifq); 1748 ifq_clr_oactive(vioq->viq_ifq); 1749 vioq->viq_txfree_slots = vioq->viq_txvq->vq_num - 1; 1750 mtx_leave(&vioq->viq_txmtx); 1751 } 1752 } 1753 1754 /* 1755 * Control vq 1756 */ 1757 1758 /* 1759 * Lock the control queue and the sc_ctrl_* structs and prepare a request. 1760 * 1761 * If this function succeeds, the caller must also call either 1762 * vio_ctrl_submit() or virtio_enqueue_abort(), in both cases followed by 1763 * vio_ctrl_finish(). 1764 */ 1765 int 1766 vio_ctrl_start(struct vio_softc *sc, uint8_t class, uint8_t cmd, int nslots, 1767 int *slotp) 1768 { 1769 struct virtio_softc *vsc = sc->sc_virtio; 1770 struct virtqueue *vq = sc->sc_ctl_vq; 1771 int r; 1772 1773 splassert(IPL_NET); 1774 1775 while (sc->sc_ctrl_inuse != FREE) { 1776 if (sc->sc_ctrl_inuse == RESET || vio_needs_reset(sc)) 1777 return ENXIO; 1778 r = tsleep_nsec(&sc->sc_ctrl_inuse, PRIBIO, "viowait", INFSLP); 1779 if (r != 0) 1780 return r; 1781 } 1782 sc->sc_ctrl_inuse = INUSE; 1783 1784 sc->sc_ctrl_cmd->class = class; 1785 sc->sc_ctrl_cmd->command = cmd; 1786 1787 r = virtio_enqueue_prep(vq, slotp); 1788 if (r != 0) 1789 panic("%s: %s virtio_enqueue_prep: control vq busy", 1790 sc->sc_dev.dv_xname, __func__); 1791 r = virtio_enqueue_reserve(vq, *slotp, nslots + 2); 1792 if (r != 0) 1793 panic("%s: %s virtio_enqueue_reserve: control vq busy", 1794 sc->sc_dev.dv_xname, __func__); 1795 1796 vio_dmamem_enqueue(vsc, sc, vq, *slotp, sc->sc_ctrl_cmd, 1797 sizeof(*sc->sc_ctrl_cmd), 1); 1798 1799 return 0; 1800 } 1801 1802 /* 1803 * Submit a control queue request and wait for the result. 1804 * 1805 * vio_ctrl_start() must have been called successfully. 1806 * After vio_ctrl_submit(), the caller may inspect the 1807 * data returned from the hypervisor. Afterwards, the caller 1808 * must always call vio_ctrl_finish(). 1809 */ 1810 int 1811 vio_ctrl_submit(struct vio_softc *sc, int slot) 1812 { 1813 struct virtio_softc *vsc = sc->sc_virtio; 1814 struct virtqueue *vq = sc->sc_ctl_vq; 1815 int r; 1816 1817 vio_dmamem_enqueue(vsc, sc, vq, slot, sc->sc_ctrl_status, 1818 sizeof(*sc->sc_ctrl_status), 0); 1819 1820 virtio_enqueue_commit(vsc, vq, slot, 1); 1821 1822 while (sc->sc_ctrl_inuse != DONE) { 1823 if (sc->sc_ctrl_inuse == RESET || vio_needs_reset(sc)) 1824 return ENXIO; 1825 r = tsleep_nsec(&sc->sc_ctrl_inuse, PRIBIO, "viodone", 1826 VIRTIO_NET_CTRL_TIMEOUT); 1827 if (r != 0) { 1828 if (r == EWOULDBLOCK) 1829 printf("%s: ctrl queue timeout\n", 1830 sc->sc_dev.dv_xname); 1831 vio_ctrl_wakeup(sc, RESET); 1832 return ENXIO; 1833 } 1834 if (cold) 1835 virtio_check_vq(sc->sc_virtio, sc->sc_ctl_vq); 1836 } 1837 1838 VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_cmd, 1839 sizeof(*sc->sc_ctrl_cmd), BUS_DMASYNC_POSTWRITE); 1840 VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_status, 1841 sizeof(*sc->sc_ctrl_status), BUS_DMASYNC_POSTREAD); 1842 1843 if (sc->sc_ctrl_status->ack != VIRTIO_NET_OK) 1844 return EIO; 1845 1846 return 0; 1847 } 1848 1849 /* 1850 * Unlock the control queue and the sc_ctrl_* structs. 1851 * 1852 * It is ok to call this function if the control queue is marked dead 1853 * due to a fatal error. 1854 */ 1855 void 1856 vio_ctrl_finish(struct vio_softc *sc) 1857 { 1858 if (sc->sc_ctrl_inuse == RESET) 1859 return; 1860 1861 vio_ctrl_wakeup(sc, FREE); 1862 } 1863 1864 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1865 int 1866 vio_ctrl_rx(struct vio_softc *sc, int cmd, int onoff) 1867 { 1868 struct virtio_softc *vsc = sc->sc_virtio; 1869 struct virtqueue *vq = sc->sc_ctl_vq; 1870 int r, slot; 1871 1872 r = vio_ctrl_start(sc, VIRTIO_NET_CTRL_RX, cmd, 1, &slot); 1873 if (r != 0) 1874 return r; 1875 1876 sc->sc_ctrl_rx->onoff = onoff; 1877 1878 vio_dmamem_enqueue(vsc, sc, vq, slot, sc->sc_ctrl_rx, 1879 sizeof(*sc->sc_ctrl_rx), 1); 1880 1881 r = vio_ctrl_submit(sc, slot); 1882 VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_rx, 1883 sizeof(*sc->sc_ctrl_rx), BUS_DMASYNC_POSTWRITE); 1884 if (r != 0) 1885 printf("%s: ctrl cmd %d failed\n", sc->sc_dev.dv_xname, cmd); 1886 1887 DPRINTF("%s: cmd %d %d: %d\n", __func__, cmd, onoff, r); 1888 1889 vio_ctrl_finish(sc); 1890 return r; 1891 } 1892 1893 /* issue a VIRTIO_NET_CTRL_MQ class command and wait for completion */ 1894 int 1895 vio_ctrl_mq(struct vio_softc *sc) 1896 { 1897 struct virtio_softc *vsc = sc->sc_virtio; 1898 struct virtqueue *vq = sc->sc_ctl_vq; 1899 int r, slot; 1900 1901 1902 r = vio_ctrl_start(sc, VIRTIO_NET_CTRL_MQ, 1903 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 1, &slot); 1904 if (r != 0) 1905 return r; 1906 1907 sc->sc_ctrl_mq_pairs->virtqueue_pairs = sc->sc_nqueues; 1908 1909 vio_dmamem_enqueue(vsc, sc, vq, slot, sc->sc_ctrl_mq_pairs, 1910 sizeof(*sc->sc_ctrl_mq_pairs), 1); 1911 1912 r = vio_ctrl_submit(sc, slot); 1913 1914 VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_mq_pairs, 1915 sizeof(*sc->sc_ctrl_mq_pairs), BUS_DMASYNC_POSTWRITE); 1916 1917 if (r != 0) 1918 printf("%s: ctrl cmd %d failed\n", sc->sc_dev.dv_xname, 1919 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET); 1920 1921 DPRINTF("%s: cmd %d %d: %d\n", __func__, 1922 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, sc->sc_nqueues, r); 1923 1924 vio_ctrl_finish(sc); 1925 return r; 1926 } 1927 1928 int 1929 vio_ctrl_guest_offloads(struct vio_softc *sc, uint64_t features) 1930 { 1931 struct virtio_softc *vsc = sc->sc_virtio; 1932 struct virtqueue *vq = sc->sc_ctl_vq; 1933 int r, slot; 1934 1935 r = vio_ctrl_start(sc, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 1936 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, 1, &slot); 1937 if (r != 0) 1938 return r; 1939 1940 sc->sc_ctrl_guest_offloads->offloads = features; 1941 1942 vio_dmamem_enqueue(vsc, sc, vq, slot, sc->sc_ctrl_guest_offloads, 1943 sizeof(*sc->sc_ctrl_guest_offloads), 1); 1944 1945 r = vio_ctrl_submit(sc, slot); 1946 1947 VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_guest_offloads, 1948 sizeof(*sc->sc_ctrl_guest_offloads), BUS_DMASYNC_POSTWRITE); 1949 1950 if (r != 0 && features != 0) { 1951 printf("%s: offload features 0x%llx failed\n", 1952 sc->sc_dev.dv_xname, features); 1953 } 1954 1955 DPRINTF("%s: offload features 0x%llx: %d\n", __func__, features, r); 1956 1957 vio_ctrl_finish(sc); 1958 return r; 1959 } 1960 1961 void 1962 vio_ctrl_wakeup(struct vio_softc *sc, enum vio_ctrl_state new) 1963 { 1964 sc->sc_ctrl_inuse = new; 1965 wakeup(&sc->sc_ctrl_inuse); 1966 } 1967 1968 int 1969 vio_ctrleof(struct virtqueue *vq) 1970 { 1971 struct virtio_softc *vsc = vq->vq_owner; 1972 struct vio_softc *sc = (struct vio_softc *)vsc->sc_child; 1973 int r = 0, ret, slot, s; 1974 1975 KERNEL_LOCK(); 1976 s = splnet(); 1977 again: 1978 ret = virtio_dequeue(vsc, vq, &slot, NULL); 1979 if (ret == ENOENT) 1980 goto out; 1981 virtio_dequeue_commit(vq, slot); 1982 r++; 1983 vio_ctrl_wakeup(sc, DONE); 1984 if (virtio_start_vq_intr(vsc, vq)) 1985 goto again; 1986 1987 out: 1988 splx(s); 1989 KERNEL_UNLOCK(); 1990 return r; 1991 } 1992 1993 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 1994 int 1995 vio_set_rx_filter(struct vio_softc *sc) 1996 { 1997 /* filter already set in sc_ctrl_mac_tbl */ 1998 struct virtio_softc *vsc = sc->sc_virtio; 1999 struct virtqueue *vq = sc->sc_ctl_vq; 2000 int r, slot; 2001 size_t len_uc, len_mc; 2002 2003 2004 r = vio_ctrl_start(sc, VIRTIO_NET_CTRL_MAC, 2005 VIRTIO_NET_CTRL_MAC_TABLE_SET, 2, &slot); 2006 if (r != 0) 2007 return r; 2008 2009 len_uc = sizeof(*sc->sc_ctrl_mac_tbl_uc) + 2010 sc->sc_ctrl_mac_tbl_uc->nentries * ETHER_ADDR_LEN; 2011 len_mc = sizeof(*sc->sc_ctrl_mac_tbl_mc) + 2012 sc->sc_ctrl_mac_tbl_mc->nentries * ETHER_ADDR_LEN; 2013 vio_dmamem_enqueue(vsc, sc, vq, slot, sc->sc_ctrl_mac_tbl_uc, len_uc, 2014 1); 2015 vio_dmamem_enqueue(vsc, sc, vq, slot, sc->sc_ctrl_mac_tbl_mc, len_mc, 2016 1); 2017 2018 r = vio_ctrl_submit(sc, slot); 2019 VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_mac_tbl_uc, len_uc, 2020 BUS_DMASYNC_POSTWRITE); 2021 VIO_DMAMEM_SYNC(vsc, sc, sc->sc_ctrl_mac_tbl_mc, len_mc, 2022 BUS_DMASYNC_POSTWRITE); 2023 2024 if (r != 0) { 2025 /* The host's filter table is not large enough */ 2026 printf("%s: failed setting rx filter\n", sc->sc_dev.dv_xname); 2027 } 2028 2029 vio_ctrl_finish(sc); 2030 return r; 2031 } 2032 2033 void 2034 vio_iff(struct vio_softc *sc) 2035 { 2036 struct virtio_softc *vsc = sc->sc_virtio; 2037 struct ifnet *ifp = &sc->sc_ac.ac_if; 2038 struct arpcom *ac = &sc->sc_ac; 2039 struct ether_multi *enm; 2040 struct ether_multistep step; 2041 int nentries = 0; 2042 int promisc = 0, allmulti = 0, rxfilter = 0; 2043 int r; 2044 2045 splassert(IPL_NET); 2046 2047 ifp->if_flags &= ~IFF_ALLMULTI; 2048 2049 if (!virtio_has_feature(vsc, VIRTIO_NET_F_CTRL_RX)) { 2050 /* no ctrl vq; always promisc */ 2051 ifp->if_flags |= IFF_ALLMULTI | IFF_PROMISC; 2052 return; 2053 } 2054 2055 if (sc->sc_dev.dv_cfdata->cf_flags & CONFFLAG_QEMU_VLAN_BUG) 2056 ifp->if_flags |= IFF_PROMISC; 2057 2058 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 2059 ac->ac_multicnt >= VIRTIO_NET_CTRL_MAC_MC_ENTRIES) { 2060 ifp->if_flags |= IFF_ALLMULTI; 2061 if (ifp->if_flags & IFF_PROMISC) 2062 promisc = 1; 2063 else 2064 allmulti = 1; 2065 } else { 2066 rxfilter = 1; 2067 2068 ETHER_FIRST_MULTI(step, ac, enm); 2069 while (enm != NULL) { 2070 memcpy(sc->sc_ctrl_mac_tbl_mc->macs[nentries++], 2071 enm->enm_addrlo, ETHER_ADDR_LEN); 2072 2073 ETHER_NEXT_MULTI(step, enm); 2074 } 2075 } 2076 2077 /* set unicast address, VirtualBox wants that */ 2078 memcpy(sc->sc_ctrl_mac_tbl_uc->macs[0], ac->ac_enaddr, ETHER_ADDR_LEN); 2079 sc->sc_ctrl_mac_tbl_uc->nentries = 1; 2080 2081 sc->sc_ctrl_mac_tbl_mc->nentries = rxfilter ? nentries : 0; 2082 2083 r = vio_set_rx_filter(sc); 2084 if (r == EIO) 2085 allmulti = 1; /* fallback */ 2086 else if (r != 0) 2087 return; 2088 2089 r = vio_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, allmulti); 2090 if (r == EIO) 2091 promisc = 1; /* fallback */ 2092 else if (r != 0) 2093 return; 2094 2095 vio_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, promisc); 2096 } 2097