1 /* $NetBSD: if_vioif.c,v 1.65 2020/05/28 23:25:17 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.65 2020/05/28 23:25:17 riastradh Exp $"); 30 31 #ifdef _KERNEL_OPT 32 #include "opt_net_mpsafe.h" 33 #endif 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/atomic.h> 39 #include <sys/bus.h> 40 #include <sys/condvar.h> 41 #include <sys/device.h> 42 #include <sys/evcnt.h> 43 #include <sys/intr.h> 44 #include <sys/kmem.h> 45 #include <sys/mbuf.h> 46 #include <sys/mutex.h> 47 #include <sys/sockio.h> 48 #include <sys/cpu.h> 49 #include <sys/module.h> 50 #include <sys/pcq.h> 51 #include <sys/workqueue.h> 52 53 #include <dev/pci/virtioreg.h> 54 #include <dev/pci/virtiovar.h> 55 56 #include <net/if.h> 57 #include <net/if_media.h> 58 #include <net/if_ether.h> 59 60 #include <net/bpf.h> 61 62 #include "ioconf.h" 63 64 #ifdef NET_MPSAFE 65 #define VIOIF_MPSAFE 1 66 #define VIOIF_MULTIQ 1 67 #endif 68 69 /* 70 * if_vioifreg.h: 71 */ 72 /* Configuration registers */ 73 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 74 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 75 #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */ 76 77 /* Feature bits */ 78 #define VIRTIO_NET_F_CSUM __BIT(0) 79 #define VIRTIO_NET_F_GUEST_CSUM __BIT(1) 80 #define VIRTIO_NET_F_MAC __BIT(5) 81 #define VIRTIO_NET_F_GSO __BIT(6) 82 #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7) 83 #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8) 84 #define VIRTIO_NET_F_GUEST_ECN __BIT(9) 85 #define VIRTIO_NET_F_GUEST_UFO __BIT(10) 86 #define VIRTIO_NET_F_HOST_TSO4 __BIT(11) 87 #define VIRTIO_NET_F_HOST_TSO6 __BIT(12) 88 #define VIRTIO_NET_F_HOST_ECN __BIT(13) 89 #define VIRTIO_NET_F_HOST_UFO __BIT(14) 90 #define VIRTIO_NET_F_MRG_RXBUF __BIT(15) 91 #define VIRTIO_NET_F_STATUS __BIT(16) 92 #define VIRTIO_NET_F_CTRL_VQ __BIT(17) 93 #define VIRTIO_NET_F_CTRL_RX __BIT(18) 94 #define VIRTIO_NET_F_CTRL_VLAN __BIT(19) 95 #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20) 96 #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21) 97 #define VIRTIO_NET_F_MQ __BIT(22) 98 99 #define VIRTIO_NET_FLAG_BITS \ 100 VIRTIO_COMMON_FLAG_BITS \ 101 "\x17""MQ" \ 102 "\x16""GUEST_ANNOUNCE" \ 103 "\x15""CTRL_RX_EXTRA" \ 104 "\x14""CTRL_VLAN" \ 105 "\x13""CTRL_RX" \ 106 "\x12""CTRL_VQ" \ 107 "\x11""STATUS" \ 108 "\x10""MRG_RXBUF" \ 109 "\x0f""HOST_UFO" \ 110 "\x0e""HOST_ECN" \ 111 "\x0d""HOST_TSO6" \ 112 "\x0c""HOST_TSO4" \ 113 "\x0b""GUEST_UFO" \ 114 "\x0a""GUEST_ECN" \ 115 "\x09""GUEST_TSO6" \ 116 "\x08""GUEST_TSO4" \ 117 "\x07""GSO" \ 118 "\x06""MAC" \ 119 "\x02""GUEST_CSUM" \ 120 "\x01""CSUM" 121 122 /* Status */ 123 #define VIRTIO_NET_S_LINK_UP 1 124 125 /* Packet header structure */ 126 struct virtio_net_hdr { 127 uint8_t flags; 128 uint8_t gso_type; 129 uint16_t hdr_len; 130 uint16_t gso_size; 131 uint16_t csum_start; 132 uint16_t csum_offset; 133 #if 0 134 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */ 135 #endif 136 } __packed; 137 138 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 139 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 140 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 141 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 142 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 143 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 144 145 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 146 147 /* Control virtqueue */ 148 struct virtio_net_ctrl_cmd { 149 uint8_t class; 150 uint8_t command; 151 } __packed; 152 #define VIRTIO_NET_CTRL_RX 0 153 # define VIRTIO_NET_CTRL_RX_PROMISC 0 154 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 155 156 #define VIRTIO_NET_CTRL_MAC 1 157 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 158 159 #define VIRTIO_NET_CTRL_VLAN 2 160 # define VIRTIO_NET_CTRL_VLAN_ADD 0 161 # define VIRTIO_NET_CTRL_VLAN_DEL 1 162 163 #define VIRTIO_NET_CTRL_MQ 4 164 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 165 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 166 # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 167 168 struct virtio_net_ctrl_status { 169 uint8_t ack; 170 } __packed; 171 #define VIRTIO_NET_OK 0 172 #define VIRTIO_NET_ERR 1 173 174 struct virtio_net_ctrl_rx { 175 uint8_t onoff; 176 } __packed; 177 178 struct virtio_net_ctrl_mac_tbl { 179 uint32_t nentries; 180 uint8_t macs[][ETHER_ADDR_LEN]; 181 } __packed; 182 183 struct virtio_net_ctrl_vlan { 184 uint16_t id; 185 } __packed; 186 187 struct virtio_net_ctrl_mq { 188 uint16_t virtqueue_pairs; 189 } __packed; 190 191 struct vioif_ctrl_cmdspec { 192 bus_dmamap_t dmamap; 193 void *buf; 194 bus_size_t bufsize; 195 }; 196 197 /* 198 * if_vioifvar.h: 199 */ 200 201 /* 202 * Locking notes: 203 * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and 204 * a field in vioif_rxqueue is protected by rxq_lock (a spin mutex). 205 * - more than one lock cannot be held at onece 206 * + ctrlq_inuse is protected by ctrlq_wait_lock. 207 * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse 208 * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock 209 * + fields in vioif_softc except queues are protected by 210 * sc->sc_lock(an adaptive mutex) 211 * - the lock is held before acquisition of other locks 212 */ 213 214 struct vioif_work { 215 struct work cookie; 216 void (*func)(void *); 217 void *arg; 218 unsigned int added; 219 }; 220 221 struct vioif_txqueue { 222 kmutex_t *txq_lock; /* lock for tx operations */ 223 224 struct virtqueue *txq_vq; 225 bool txq_stopping; 226 bool txq_link_active; 227 pcq_t *txq_intrq; 228 229 struct virtio_net_hdr *txq_hdrs; 230 bus_dmamap_t *txq_hdr_dmamaps; 231 232 struct mbuf **txq_mbufs; 233 bus_dmamap_t *txq_dmamaps; 234 235 void *txq_deferred_transmit; 236 void *txq_handle_si; 237 struct vioif_work txq_work; 238 bool txq_workqueue; 239 bool txq_active; 240 241 char txq_evgroup[16]; 242 struct evcnt txq_defrag_failed; 243 struct evcnt txq_mbuf_load_failed; 244 struct evcnt txq_enqueue_reserve_failed; 245 }; 246 247 struct vioif_rxqueue { 248 kmutex_t *rxq_lock; /* lock for rx operations */ 249 250 struct virtqueue *rxq_vq; 251 bool rxq_stopping; 252 253 struct virtio_net_hdr *rxq_hdrs; 254 bus_dmamap_t *rxq_hdr_dmamaps; 255 256 struct mbuf **rxq_mbufs; 257 bus_dmamap_t *rxq_dmamaps; 258 259 void *rxq_handle_si; 260 struct vioif_work rxq_work; 261 bool rxq_workqueue; 262 bool rxq_active; 263 264 char rxq_evgroup[16]; 265 struct evcnt rxq_mbuf_add_failed; 266 }; 267 268 struct vioif_ctrlqueue { 269 struct virtqueue *ctrlq_vq; 270 enum { 271 FREE, INUSE, DONE 272 } ctrlq_inuse; 273 kcondvar_t ctrlq_wait; 274 kmutex_t ctrlq_wait_lock; 275 struct lwp *ctrlq_owner; 276 277 struct virtio_net_ctrl_cmd *ctrlq_cmd; 278 struct virtio_net_ctrl_status *ctrlq_status; 279 struct virtio_net_ctrl_rx *ctrlq_rx; 280 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc; 281 struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc; 282 struct virtio_net_ctrl_mq *ctrlq_mq; 283 284 bus_dmamap_t ctrlq_cmd_dmamap; 285 bus_dmamap_t ctrlq_status_dmamap; 286 bus_dmamap_t ctrlq_rx_dmamap; 287 bus_dmamap_t ctrlq_tbl_uc_dmamap; 288 bus_dmamap_t ctrlq_tbl_mc_dmamap; 289 bus_dmamap_t ctrlq_mq_dmamap; 290 291 struct evcnt ctrlq_cmd_load_failed; 292 struct evcnt ctrlq_cmd_failed; 293 }; 294 295 struct vioif_softc { 296 device_t sc_dev; 297 kmutex_t sc_lock; 298 struct sysctllog *sc_sysctllog; 299 300 struct virtio_softc *sc_virtio; 301 struct virtqueue *sc_vqs; 302 303 int sc_max_nvq_pairs; 304 int sc_req_nvq_pairs; 305 int sc_act_nvq_pairs; 306 307 uint8_t sc_mac[ETHER_ADDR_LEN]; 308 struct ethercom sc_ethercom; 309 short sc_deferred_init_done; 310 bool sc_link_active; 311 312 struct vioif_txqueue *sc_txq; 313 struct vioif_rxqueue *sc_rxq; 314 315 bool sc_has_ctrl; 316 struct vioif_ctrlqueue sc_ctrlq; 317 318 bus_dma_segment_t sc_hdr_segs[1]; 319 void *sc_dmamem; 320 void *sc_kmem; 321 322 void *sc_ctl_softint; 323 324 struct workqueue *sc_txrx_workqueue; 325 bool sc_txrx_workqueue_sysctl; 326 u_int sc_tx_intr_process_limit; 327 u_int sc_tx_process_limit; 328 u_int sc_rx_intr_process_limit; 329 u_int sc_rx_process_limit; 330 }; 331 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 332 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 333 334 #define VIOIF_TX_INTR_PROCESS_LIMIT 256 335 #define VIOIF_TX_PROCESS_LIMIT 256 336 #define VIOIF_RX_INTR_PROCESS_LIMIT 0U 337 #define VIOIF_RX_PROCESS_LIMIT 256 338 339 #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET 340 341 /* cfattach interface functions */ 342 static int vioif_match(device_t, cfdata_t, void *); 343 static void vioif_attach(device_t, device_t, void *); 344 static void vioif_deferred_init(device_t); 345 static int vioif_finalize_teardown(device_t); 346 347 /* ifnet interface functions */ 348 static int vioif_init(struct ifnet *); 349 static void vioif_stop(struct ifnet *, int); 350 static void vioif_start(struct ifnet *); 351 static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *); 352 static int vioif_transmit(struct ifnet *, struct mbuf *); 353 static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *); 354 static int vioif_ioctl(struct ifnet *, u_long, void *); 355 static void vioif_watchdog(struct ifnet *); 356 357 /* rx */ 358 static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int); 359 static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int); 360 static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *); 361 static void vioif_rx_queue_clear(struct vioif_rxqueue *); 362 static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *, 363 struct vioif_rxqueue *, u_int); 364 static int vioif_rx_intr(void *); 365 static void vioif_rx_handle(void *); 366 static void vioif_rx_sched_handle(struct vioif_softc *, 367 struct vioif_rxqueue *); 368 static void vioif_rx_drain(struct vioif_rxqueue *); 369 370 /* tx */ 371 static int vioif_tx_intr(void *); 372 static void vioif_tx_handle(void *); 373 static void vioif_tx_sched_handle(struct vioif_softc *, 374 struct vioif_txqueue *); 375 static void vioif_tx_queue_clear(struct vioif_txqueue *); 376 static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *, 377 struct vioif_txqueue *, u_int); 378 static void vioif_tx_drain(struct vioif_txqueue *); 379 static void vioif_deferred_transmit(void *); 380 381 /* workqueue */ 382 static struct workqueue* 383 vioif_workq_create(const char *, pri_t, int, int); 384 static void vioif_workq_destroy(struct workqueue *); 385 static void vioif_workq_work(struct work *, void *); 386 static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); 387 static void vioif_work_add(struct workqueue *, struct vioif_work *); 388 static void vioif_work_wait(struct workqueue *, struct vioif_work *); 389 390 /* other control */ 391 static bool vioif_is_link_up(struct vioif_softc *); 392 static void vioif_update_link_status(struct vioif_softc *); 393 static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 394 static int vioif_set_promisc(struct vioif_softc *, bool); 395 static int vioif_set_allmulti(struct vioif_softc *, bool); 396 static int vioif_set_rx_filter(struct vioif_softc *); 397 static int vioif_rx_filter(struct vioif_softc *); 398 static int vioif_ctrl_intr(void *); 399 static int vioif_config_change(struct virtio_softc *); 400 static void vioif_ctl_softint(void *); 401 static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); 402 static void vioif_enable_interrupt_vqpairs(struct vioif_softc *); 403 static void vioif_disable_interrupt_vqpairs(struct vioif_softc *); 404 static int vioif_setup_sysctl(struct vioif_softc *); 405 static void vioif_setup_stats(struct vioif_softc *); 406 407 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 408 vioif_match, vioif_attach, NULL, NULL); 409 410 static int 411 vioif_match(device_t parent, cfdata_t match, void *aux) 412 { 413 struct virtio_attach_args *va = aux; 414 415 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK) 416 return 1; 417 418 return 0; 419 } 420 421 static int 422 vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, 423 bus_size_t size, int nsegs, const char *usage) 424 { 425 int r; 426 427 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, 428 nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); 429 430 if (r != 0) { 431 aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " 432 "error code %d\n", usage, r); 433 } 434 435 return r; 436 } 437 438 static void 439 vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) 440 { 441 442 if (*map) { 443 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); 444 *map = NULL; 445 } 446 } 447 448 static int 449 vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, 450 void *buf, bus_size_t size, int nsegs, int rw, const char *usage) 451 { 452 int r; 453 454 r = vioif_dmamap_create(sc, map, size, nsegs, usage); 455 if (r != 0) 456 return 1; 457 458 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, 459 size, NULL, rw | BUS_DMA_NOWAIT); 460 if (r != 0) { 461 vioif_dmamap_destroy(sc, map); 462 aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " 463 "error code %d\n", usage, r); 464 } 465 466 return r; 467 } 468 469 static void * 470 vioif_assign_mem(intptr_t *p, size_t size) 471 { 472 intptr_t rv; 473 474 rv = *p; 475 *p += size; 476 477 return (void *)rv; 478 } 479 480 static void 481 vioif_alloc_queues(struct vioif_softc *sc) 482 { 483 int nvq_pairs = sc->sc_max_nvq_pairs; 484 int nvqs = nvq_pairs * 2; 485 int i; 486 487 KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); 488 489 sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs, 490 KM_SLEEP); 491 sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs, 492 KM_SLEEP); 493 494 if (sc->sc_has_ctrl) 495 nvqs++; 496 497 sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); 498 nvqs = 0; 499 for (i = 0; i < nvq_pairs; i++) { 500 sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++]; 501 sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++]; 502 } 503 504 if (sc->sc_has_ctrl) 505 sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++]; 506 } 507 508 static void 509 vioif_free_queues(struct vioif_softc *sc) 510 { 511 int nvq_pairs = sc->sc_max_nvq_pairs; 512 int nvqs = nvq_pairs * 2; 513 514 if (sc->sc_ctrlq.ctrlq_vq) 515 nvqs++; 516 517 if (sc->sc_txq) { 518 kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs); 519 sc->sc_txq = NULL; 520 } 521 522 if (sc->sc_rxq) { 523 kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs); 524 sc->sc_rxq = NULL; 525 } 526 527 if (sc->sc_vqs) { 528 kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); 529 sc->sc_vqs = NULL; 530 } 531 } 532 533 /* allocate memory */ 534 /* 535 * dma memory is used for: 536 * rxq_hdrs[slot]: metadata array for received frames (READ) 537 * txq_hdrs[slot]: metadata array for frames to be sent (WRITE) 538 * ctrlq_cmd: command to be sent via ctrl vq (WRITE) 539 * ctrlq_status: return value for a command via ctrl vq (READ) 540 * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command 541 * (WRITE) 542 * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 543 * class command (WRITE) 544 * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 545 * class command (WRITE) 546 * ctrlq_* structures are allocated only one each; they are protected by 547 * ctrlq_inuse variable and ctrlq_wait condvar. 548 */ 549 /* 550 * dynamically allocated memory is used for: 551 * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] 552 * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] 553 * rxq_dmamaps[slot]: bus_dmamap_t array for received payload 554 * txq_dmamaps[slot]: bus_dmamap_t array for sent payload 555 * rxq_mbufs[slot]: mbuf pointer array for received frames 556 * txq_mbufs[slot]: mbuf pointer array for sent frames 557 */ 558 static int 559 vioif_alloc_mems(struct vioif_softc *sc) 560 { 561 struct virtio_softc *vsc = sc->sc_virtio; 562 struct vioif_txqueue *txq; 563 struct vioif_rxqueue *rxq; 564 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 565 int allocsize, allocsize2, r, rsegs, i, qid; 566 void *vaddr; 567 intptr_t p; 568 569 allocsize = 0; 570 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 571 rxq = &sc->sc_rxq[qid]; 572 txq = &sc->sc_txq[qid]; 573 574 allocsize += 575 sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num; 576 allocsize += 577 sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num; 578 } 579 if (sc->sc_has_ctrl) { 580 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 581 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 582 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 583 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) 584 + sizeof(struct virtio_net_ctrl_mac_tbl) 585 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 586 allocsize += sizeof(struct virtio_net_ctrl_mq) * 1; 587 } 588 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, 589 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 590 if (r != 0) { 591 aprint_error_dev(sc->sc_dev, 592 "DMA memory allocation failed, size %d, " 593 "error code %d\n", allocsize, r); 594 goto err_none; 595 } 596 r = bus_dmamem_map(virtio_dmat(vsc), 597 &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT); 598 if (r != 0) { 599 aprint_error_dev(sc->sc_dev, 600 "DMA memory map failed, error code %d\n", r); 601 goto err_dmamem_alloc; 602 } 603 604 memset(vaddr, 0, allocsize); 605 sc->sc_dmamem = vaddr; 606 p = (intptr_t) vaddr; 607 608 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 609 rxq = &sc->sc_rxq[qid]; 610 txq = &sc->sc_txq[qid]; 611 612 rxq->rxq_hdrs = vioif_assign_mem(&p, 613 sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num); 614 txq->txq_hdrs = vioif_assign_mem(&p, 615 sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num); 616 } 617 if (sc->sc_has_ctrl) { 618 ctrlq->ctrlq_cmd = vioif_assign_mem(&p, 619 sizeof(*ctrlq->ctrlq_cmd)); 620 ctrlq->ctrlq_status = vioif_assign_mem(&p, 621 sizeof(*ctrlq->ctrlq_status)); 622 ctrlq->ctrlq_rx = vioif_assign_mem(&p, 623 sizeof(*ctrlq->ctrlq_rx)); 624 ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, 625 sizeof(*ctrlq->ctrlq_mac_tbl_uc)); 626 ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, 627 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 628 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); 629 ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); 630 } 631 632 allocsize2 = 0; 633 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 634 int rxqsize, txqsize; 635 636 rxq = &sc->sc_rxq[qid]; 637 txq = &sc->sc_txq[qid]; 638 rxqsize = rxq->rxq_vq->vq_num; 639 txqsize = txq->txq_vq->vq_num; 640 641 allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize; 642 allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize; 643 allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize; 644 645 allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize; 646 allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize; 647 allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize; 648 } 649 vaddr = kmem_zalloc(allocsize2, KM_SLEEP); 650 sc->sc_kmem = vaddr; 651 p = (intptr_t) vaddr; 652 653 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 654 int rxqsize, txqsize; 655 rxq = &sc->sc_rxq[qid]; 656 txq = &sc->sc_txq[qid]; 657 rxqsize = rxq->rxq_vq->vq_num; 658 txqsize = txq->txq_vq->vq_num; 659 660 rxq->rxq_hdr_dmamaps = vioif_assign_mem(&p, 661 sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize); 662 txq->txq_hdr_dmamaps = vioif_assign_mem(&p, 663 sizeof(txq->txq_hdr_dmamaps[0]) * txqsize); 664 rxq->rxq_dmamaps = vioif_assign_mem(&p, 665 sizeof(rxq->rxq_dmamaps[0]) * rxqsize); 666 txq->txq_dmamaps = vioif_assign_mem(&p, 667 sizeof(txq->txq_dmamaps[0]) * txqsize); 668 rxq->rxq_mbufs = vioif_assign_mem(&p, 669 sizeof(rxq->rxq_mbufs[0]) * rxqsize); 670 txq->txq_mbufs = vioif_assign_mem(&p, 671 sizeof(txq->txq_mbufs[0]) * txqsize); 672 } 673 674 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 675 rxq = &sc->sc_rxq[qid]; 676 txq = &sc->sc_txq[qid]; 677 678 for (i = 0; i < rxq->rxq_vq->vq_num; i++) { 679 r = vioif_dmamap_create_load(sc, &rxq->rxq_hdr_dmamaps[i], 680 &rxq->rxq_hdrs[i], sizeof(rxq->rxq_hdrs[0]), 1, 681 BUS_DMA_READ, "rx header"); 682 if (r != 0) 683 goto err_reqs; 684 685 r = vioif_dmamap_create(sc, &rxq->rxq_dmamaps[i], 686 MCLBYTES, 1, "rx payload"); 687 if (r != 0) 688 goto err_reqs; 689 } 690 691 for (i = 0; i < txq->txq_vq->vq_num; i++) { 692 r = vioif_dmamap_create_load(sc, &txq->txq_hdr_dmamaps[i], 693 &txq->txq_hdrs[i], sizeof(txq->txq_hdrs[0]), 1, 694 BUS_DMA_READ, "tx header"); 695 if (r != 0) 696 goto err_reqs; 697 698 r = vioif_dmamap_create(sc, &txq->txq_dmamaps[i], ETHER_MAX_LEN, 699 VIRTIO_NET_TX_MAXNSEGS, "tx payload"); 700 if (r != 0) 701 goto err_reqs; 702 } 703 } 704 705 if (sc->sc_has_ctrl) { 706 /* control vq class & command */ 707 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, 708 ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, 709 BUS_DMA_WRITE, "control command"); 710 if (r != 0) 711 goto err_reqs; 712 713 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, 714 ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, 715 BUS_DMA_READ, "control status"); 716 if (r != 0) 717 goto err_reqs; 718 719 /* control vq rx mode command parameter */ 720 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, 721 ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, 722 BUS_DMA_WRITE, "rx mode control command"); 723 if (r != 0) 724 goto err_reqs; 725 726 /* multiqueue set command */ 727 r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, 728 ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, 729 BUS_DMA_WRITE, "multiqueue set command"); 730 if (r != 0) 731 goto err_reqs; 732 733 /* control vq MAC filter table for unicast */ 734 /* do not load now since its length is variable */ 735 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, 736 sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1, 737 "unicast MAC address filter command"); 738 if (r != 0) 739 goto err_reqs; 740 741 /* control vq MAC filter table for multicast */ 742 r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, 743 sizeof(*ctrlq->ctrlq_mac_tbl_mc) 744 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, 745 "multicast MAC address filter command"); 746 } 747 748 return 0; 749 750 err_reqs: 751 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); 752 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); 753 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); 754 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); 755 vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); 756 for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) { 757 rxq = &sc->sc_rxq[qid]; 758 txq = &sc->sc_txq[qid]; 759 760 for (i = 0; i < txq->txq_vq->vq_num; i++) { 761 vioif_dmamap_destroy(sc, &txq->txq_dmamaps[i]); 762 vioif_dmamap_destroy(sc, &txq->txq_hdr_dmamaps[i]); 763 } 764 for (i = 0; i < rxq->rxq_vq->vq_num; i++) { 765 vioif_dmamap_destroy(sc, &rxq->rxq_dmamaps[i]); 766 vioif_dmamap_destroy(sc, &rxq->rxq_hdr_dmamaps[i]); 767 } 768 } 769 if (sc->sc_kmem) { 770 kmem_free(sc->sc_kmem, allocsize2); 771 sc->sc_kmem = NULL; 772 } 773 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize); 774 err_dmamem_alloc: 775 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); 776 err_none: 777 return -1; 778 } 779 780 static void 781 vioif_attach(device_t parent, device_t self, void *aux) 782 { 783 struct vioif_softc *sc = device_private(self); 784 struct virtio_softc *vsc = device_private(parent); 785 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 786 struct vioif_txqueue *txq; 787 struct vioif_rxqueue *rxq; 788 uint32_t features, req_features; 789 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 790 u_int softint_flags; 791 int r, i, nvqs=0, req_flags; 792 char xnamebuf[MAXCOMLEN]; 793 794 if (virtio_child(vsc) != NULL) { 795 aprint_normal(": child already attached for %s; " 796 "something wrong...\n", device_xname(parent)); 797 return; 798 } 799 800 sc->sc_dev = self; 801 sc->sc_virtio = vsc; 802 sc->sc_link_active = false; 803 804 sc->sc_max_nvq_pairs = 1; 805 sc->sc_req_nvq_pairs = 1; 806 sc->sc_act_nvq_pairs = 1; 807 sc->sc_txrx_workqueue_sysctl = true; 808 sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; 809 sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; 810 sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; 811 sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; 812 813 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); 814 815 snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); 816 sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, 817 IPL_NET, WQ_PERCPU | WQ_MPSAFE); 818 if (sc->sc_txrx_workqueue == NULL) 819 goto err; 820 821 req_flags = 0; 822 823 #ifdef VIOIF_MPSAFE 824 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE; 825 #endif 826 req_flags |= VIRTIO_F_PCI_INTR_MSIX; 827 828 req_features = 829 VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 830 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; 831 #ifdef VIOIF_MULTIQ 832 req_features |= VIRTIO_NET_F_MQ; 833 #endif 834 virtio_child_attach_start(vsc, self, IPL_NET, NULL, 835 vioif_config_change, virtio_vq_intrhand, req_flags, 836 req_features, VIRTIO_NET_FLAG_BITS); 837 838 features = virtio_features(vsc); 839 840 if (features & VIRTIO_NET_F_MAC) { 841 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 842 sc->sc_mac[i] = virtio_read_device_config_1(vsc, 843 VIRTIO_NET_CONFIG_MAC + i); 844 } 845 } else { 846 /* code stolen from sys/net/if_tap.c */ 847 struct timeval tv; 848 uint32_t ui; 849 getmicrouptime(&tv); 850 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 851 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 852 for (i = 0; i < __arraycount(sc->sc_mac); i++) { 853 virtio_write_device_config_1(vsc, 854 VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); 855 } 856 } 857 858 aprint_normal_dev(self, "Ethernet address %s\n", 859 ether_sprintf(sc->sc_mac)); 860 861 if ((features & VIRTIO_NET_F_CTRL_VQ) && 862 (features & VIRTIO_NET_F_CTRL_RX)) { 863 sc->sc_has_ctrl = true; 864 865 cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); 866 mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); 867 ctrlq->ctrlq_inuse = FREE; 868 } else { 869 sc->sc_has_ctrl = false; 870 } 871 872 if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { 873 sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, 874 VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); 875 876 if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) 877 goto err; 878 879 /* Limit the number of queue pairs to use */ 880 sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); 881 } 882 883 vioif_alloc_queues(sc); 884 virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); 885 886 #ifdef VIOIF_MPSAFE 887 softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; 888 #else 889 softint_flags = SOFTINT_NET; 890 #endif 891 892 /* 893 * Allocating virtqueues 894 */ 895 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 896 rxq = &sc->sc_rxq[i]; 897 txq = &sc->sc_txq[i]; 898 char qname[32]; 899 900 rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 901 902 rxq->rxq_handle_si = softint_establish(softint_flags, 903 vioif_rx_handle, rxq); 904 if (rxq->rxq_handle_si == NULL) { 905 aprint_error_dev(self, "cannot establish rx softint\n"); 906 goto err; 907 } 908 909 snprintf(qname, sizeof(qname), "rx%d", i); 910 r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs, 911 MCLBYTES+sizeof(struct virtio_net_hdr), 2, qname); 912 if (r != 0) 913 goto err; 914 nvqs++; 915 rxq->rxq_vq->vq_intrhand = vioif_rx_intr; 916 rxq->rxq_vq->vq_intrhand_arg = (void *)rxq; 917 rxq->rxq_stopping = true; 918 vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq); 919 920 txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 921 922 txq->txq_deferred_transmit = softint_establish(softint_flags, 923 vioif_deferred_transmit, txq); 924 if (txq->txq_deferred_transmit == NULL) { 925 aprint_error_dev(self, "cannot establish tx softint\n"); 926 goto err; 927 } 928 txq->txq_handle_si = softint_establish(softint_flags, 929 vioif_tx_handle, txq); 930 if (txq->txq_handle_si == NULL) { 931 aprint_error_dev(self, "cannot establish tx softint\n"); 932 goto err; 933 } 934 935 snprintf(qname, sizeof(qname), "tx%d", i); 936 r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs, 937 sizeof(struct virtio_net_hdr) 938 + (ETHER_MAX_LEN - ETHER_HDR_LEN), 939 VIRTIO_NET_TX_MAXNSEGS + 1, qname); 940 if (r != 0) 941 goto err; 942 nvqs++; 943 txq->txq_vq->vq_intrhand = vioif_tx_intr; 944 txq->txq_vq->vq_intrhand_arg = (void *)txq; 945 txq->txq_link_active = sc->sc_link_active; 946 txq->txq_stopping = false; 947 txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP); 948 vioif_work_set(&txq->txq_work, vioif_tx_handle, txq); 949 } 950 951 if (sc->sc_has_ctrl) { 952 /* 953 * Allocating a virtqueue for control channel 954 */ 955 r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs, 956 NBPG, 1, "control"); 957 if (r != 0) { 958 aprint_error_dev(self, "failed to allocate " 959 "a virtqueue for control channel, error code %d\n", 960 r); 961 962 sc->sc_has_ctrl = false; 963 cv_destroy(&ctrlq->ctrlq_wait); 964 mutex_destroy(&ctrlq->ctrlq_wait_lock); 965 } else { 966 nvqs++; 967 ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; 968 ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; 969 } 970 } 971 972 sc->sc_ctl_softint = softint_establish(softint_flags, 973 vioif_ctl_softint, sc); 974 if (sc->sc_ctl_softint == NULL) { 975 aprint_error_dev(self, "cannot establish ctl softint\n"); 976 goto err; 977 } 978 979 if (vioif_alloc_mems(sc) < 0) 980 goto err; 981 982 if (virtio_child_attach_finish(vsc) != 0) 983 goto err; 984 985 if (vioif_setup_sysctl(sc) != 0) { 986 aprint_error_dev(self, "unable to create sysctl node\n"); 987 /* continue */ 988 } 989 990 vioif_setup_stats(sc); 991 992 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 993 ifp->if_softc = sc; 994 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 995 #ifdef VIOIF_MPSAFE 996 ifp->if_extflags = IFEF_MPSAFE; 997 #endif 998 ifp->if_start = vioif_start; 999 if (sc->sc_req_nvq_pairs > 1) 1000 ifp->if_transmit = vioif_transmit; 1001 ifp->if_ioctl = vioif_ioctl; 1002 ifp->if_init = vioif_init; 1003 ifp->if_stop = vioif_stop; 1004 ifp->if_capabilities = 0; 1005 ifp->if_watchdog = vioif_watchdog; 1006 txq = &sc->sc_txq[0]; 1007 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN)); 1008 IFQ_SET_READY(&ifp->if_snd); 1009 1010 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 1011 1012 if_attach(ifp); 1013 if_deferred_start_init(ifp, NULL); 1014 ether_ifattach(ifp, sc->sc_mac); 1015 1016 return; 1017 1018 err: 1019 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 1020 rxq = &sc->sc_rxq[i]; 1021 txq = &sc->sc_txq[i]; 1022 1023 if (rxq->rxq_lock) { 1024 mutex_obj_free(rxq->rxq_lock); 1025 rxq->rxq_lock = NULL; 1026 } 1027 1028 if (rxq->rxq_handle_si) { 1029 softint_disestablish(rxq->rxq_handle_si); 1030 rxq->rxq_handle_si = NULL; 1031 } 1032 1033 if (txq->txq_lock) { 1034 mutex_obj_free(txq->txq_lock); 1035 txq->txq_lock = NULL; 1036 } 1037 1038 if (txq->txq_handle_si) { 1039 softint_disestablish(txq->txq_handle_si); 1040 txq->txq_handle_si = NULL; 1041 } 1042 1043 if (txq->txq_deferred_transmit) { 1044 softint_disestablish(txq->txq_deferred_transmit); 1045 txq->txq_deferred_transmit = NULL; 1046 } 1047 1048 if (txq->txq_intrq) { 1049 pcq_destroy(txq->txq_intrq); 1050 txq->txq_intrq = NULL; 1051 } 1052 } 1053 1054 if (sc->sc_has_ctrl) { 1055 cv_destroy(&ctrlq->ctrlq_wait); 1056 mutex_destroy(&ctrlq->ctrlq_wait_lock); 1057 } 1058 1059 while (nvqs > 0) 1060 virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]); 1061 1062 vioif_free_queues(sc); 1063 mutex_destroy(&sc->sc_lock); 1064 virtio_child_attach_failed(vsc); 1065 config_finalize_register(self, vioif_finalize_teardown); 1066 1067 return; 1068 } 1069 1070 static int 1071 vioif_finalize_teardown(device_t self) 1072 { 1073 struct vioif_softc *sc = device_private(self); 1074 1075 if (sc->sc_txrx_workqueue != NULL) { 1076 vioif_workq_destroy(sc->sc_txrx_workqueue); 1077 sc->sc_txrx_workqueue = NULL; 1078 } 1079 1080 return 0; 1081 } 1082 1083 /* we need interrupts to make promiscuous mode off */ 1084 static void 1085 vioif_deferred_init(device_t self) 1086 { 1087 struct vioif_softc *sc = device_private(self); 1088 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1089 int r; 1090 1091 if (ifp->if_flags & IFF_PROMISC) 1092 return; 1093 1094 r = vioif_set_promisc(sc, false); 1095 if (r != 0) 1096 aprint_error_dev(self, "resetting promisc mode failed, " 1097 "error code %d\n", r); 1098 } 1099 1100 static void 1101 vioif_enable_interrupt_vqpairs(struct vioif_softc *sc) 1102 { 1103 struct virtio_softc *vsc = sc->sc_virtio; 1104 struct vioif_txqueue *txq; 1105 struct vioif_rxqueue *rxq; 1106 int i; 1107 1108 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1109 txq = &sc->sc_txq[i]; 1110 rxq = &sc->sc_rxq[i]; 1111 1112 virtio_start_vq_intr(vsc, txq->txq_vq); 1113 virtio_start_vq_intr(vsc, rxq->rxq_vq); 1114 } 1115 } 1116 1117 static void 1118 vioif_disable_interrupt_vqpairs(struct vioif_softc *sc) 1119 { 1120 struct virtio_softc *vsc = sc->sc_virtio; 1121 struct vioif_txqueue *txq; 1122 struct vioif_rxqueue *rxq; 1123 int i; 1124 1125 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1126 rxq = &sc->sc_rxq[i]; 1127 txq = &sc->sc_txq[i]; 1128 1129 virtio_stop_vq_intr(vsc, rxq->rxq_vq); 1130 virtio_stop_vq_intr(vsc, txq->txq_vq); 1131 } 1132 } 1133 1134 /* 1135 * Interface functions for ifnet 1136 */ 1137 static int 1138 vioif_init(struct ifnet *ifp) 1139 { 1140 struct vioif_softc *sc = ifp->if_softc; 1141 struct virtio_softc *vsc = sc->sc_virtio; 1142 struct vioif_rxqueue *rxq; 1143 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1144 int r, i; 1145 1146 vioif_stop(ifp, 0); 1147 1148 virtio_reinit_start(vsc); 1149 virtio_negotiate_features(vsc, virtio_features(vsc)); 1150 1151 for (i = 0; i < sc->sc_req_nvq_pairs; i++) { 1152 rxq = &sc->sc_rxq[i]; 1153 1154 /* Have to set false before vioif_populate_rx_mbufs */ 1155 mutex_enter(rxq->rxq_lock); 1156 rxq->rxq_stopping = false; 1157 vioif_populate_rx_mbufs_locked(rxq); 1158 mutex_exit(rxq->rxq_lock); 1159 1160 } 1161 1162 virtio_reinit_end(vsc); 1163 1164 if (sc->sc_has_ctrl) 1165 virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); 1166 1167 r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); 1168 if (r == 0) 1169 sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; 1170 else 1171 sc->sc_act_nvq_pairs = 1; 1172 1173 for (i = 0; i < sc->sc_act_nvq_pairs; i++) 1174 sc->sc_txq[i].txq_stopping = false; 1175 1176 vioif_enable_interrupt_vqpairs(sc); 1177 1178 if (!sc->sc_deferred_init_done) { 1179 sc->sc_deferred_init_done = 1; 1180 if (sc->sc_has_ctrl) 1181 vioif_deferred_init(sc->sc_dev); 1182 } 1183 1184 vioif_update_link_status(sc); 1185 ifp->if_flags |= IFF_RUNNING; 1186 ifp->if_flags &= ~IFF_OACTIVE; 1187 vioif_rx_filter(sc); 1188 1189 return 0; 1190 } 1191 1192 static void 1193 vioif_stop(struct ifnet *ifp, int disable) 1194 { 1195 struct vioif_softc *sc = ifp->if_softc; 1196 struct virtio_softc *vsc = sc->sc_virtio; 1197 struct vioif_txqueue *txq; 1198 struct vioif_rxqueue *rxq; 1199 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1200 int i; 1201 1202 /* Take the locks to ensure that ongoing TX/RX finish */ 1203 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1204 txq = &sc->sc_txq[i]; 1205 rxq = &sc->sc_rxq[i]; 1206 1207 mutex_enter(rxq->rxq_lock); 1208 rxq->rxq_stopping = true; 1209 mutex_exit(rxq->rxq_lock); 1210 1211 mutex_enter(txq->txq_lock); 1212 txq->txq_stopping = true; 1213 mutex_exit(txq->txq_lock); 1214 } 1215 1216 /* disable interrupts */ 1217 vioif_disable_interrupt_vqpairs(sc); 1218 1219 if (sc->sc_has_ctrl) 1220 virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); 1221 1222 /* only way to stop I/O and DMA is resetting... */ 1223 virtio_reset(vsc); 1224 1225 /* rendezvous for finish of handlers */ 1226 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1227 txq = &sc->sc_txq[i]; 1228 rxq = &sc->sc_rxq[i]; 1229 1230 mutex_enter(rxq->rxq_lock); 1231 mutex_exit(rxq->rxq_lock); 1232 vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work); 1233 1234 mutex_enter(txq->txq_lock); 1235 mutex_exit(txq->txq_lock); 1236 vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work); 1237 } 1238 1239 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1240 vioif_rx_queue_clear(&sc->sc_rxq[i]); 1241 vioif_tx_queue_clear(&sc->sc_txq[i]); 1242 } 1243 1244 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1245 sc->sc_link_active = false; 1246 1247 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1248 txq = &sc->sc_txq[i]; 1249 rxq = &sc->sc_rxq[i]; 1250 1251 txq->txq_link_active = false; 1252 1253 if (disable) 1254 vioif_rx_drain(rxq); 1255 1256 vioif_tx_drain(txq); 1257 } 1258 } 1259 1260 static void 1261 vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq, 1262 bool is_transmit) 1263 { 1264 struct vioif_softc *sc = ifp->if_softc; 1265 struct virtio_softc *vsc = sc->sc_virtio; 1266 struct virtqueue *vq = txq->txq_vq; 1267 struct mbuf *m; 1268 int queued = 0; 1269 1270 KASSERT(mutex_owned(txq->txq_lock)); 1271 1272 if ((ifp->if_flags & IFF_RUNNING) == 0) 1273 return; 1274 1275 if (!txq->txq_link_active || txq->txq_stopping) 1276 return; 1277 1278 if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit) 1279 return; 1280 1281 for (;;) { 1282 int slot, r; 1283 1284 if (is_transmit) 1285 m = pcq_get(txq->txq_intrq); 1286 else 1287 IFQ_DEQUEUE(&ifp->if_snd, m); 1288 1289 if (m == NULL) 1290 break; 1291 1292 r = virtio_enqueue_prep(vsc, vq, &slot); 1293 if (r == EAGAIN) { 1294 ifp->if_flags |= IFF_OACTIVE; 1295 m_freem(m); 1296 break; 1297 } 1298 if (r != 0) 1299 panic("enqueue_prep for a tx buffer"); 1300 1301 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1302 txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1303 if (r != 0) { 1304 /* maybe just too fragmented */ 1305 struct mbuf *newm; 1306 1307 newm = m_defrag(m, M_NOWAIT); 1308 if (newm == NULL) { 1309 txq->txq_defrag_failed.ev_count++; 1310 goto skip; 1311 } 1312 1313 m = newm; 1314 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1315 txq->txq_dmamaps[slot], m, 1316 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1317 if (r != 0) { 1318 txq->txq_mbuf_load_failed.ev_count++; 1319 skip: 1320 m_freem(m); 1321 virtio_enqueue_abort(vsc, vq, slot); 1322 continue; 1323 } 1324 } 1325 1326 /* This should actually never fail */ 1327 r = virtio_enqueue_reserve(vsc, vq, slot, 1328 txq->txq_dmamaps[slot]->dm_nsegs + 1); 1329 if (r != 0) { 1330 txq->txq_enqueue_reserve_failed.ev_count++; 1331 bus_dmamap_unload(virtio_dmat(vsc), 1332 txq->txq_dmamaps[slot]); 1333 /* slot already freed by virtio_enqueue_reserve */ 1334 m_freem(m); 1335 continue; 1336 } 1337 1338 txq->txq_mbufs[slot] = m; 1339 1340 memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); 1341 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 1342 0, txq->txq_dmamaps[slot]->dm_mapsize, 1343 BUS_DMASYNC_PREWRITE); 1344 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 1345 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize, 1346 BUS_DMASYNC_PREWRITE); 1347 virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true); 1348 virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true); 1349 virtio_enqueue_commit(vsc, vq, slot, false); 1350 1351 queued++; 1352 bpf_mtap(ifp, m, BPF_D_OUT); 1353 } 1354 1355 if (queued > 0) { 1356 virtio_enqueue_commit(vsc, vq, -1, true); 1357 ifp->if_timer = 5; 1358 } 1359 } 1360 1361 static void 1362 vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq) 1363 { 1364 1365 /* 1366 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. 1367 */ 1368 vioif_send_common_locked(ifp, txq, false); 1369 1370 } 1371 1372 static void 1373 vioif_start(struct ifnet *ifp) 1374 { 1375 struct vioif_softc *sc = ifp->if_softc; 1376 struct vioif_txqueue *txq = &sc->sc_txq[0]; 1377 1378 #ifdef VIOIF_MPSAFE 1379 KASSERT(if_is_mpsafe(ifp)); 1380 #endif 1381 1382 mutex_enter(txq->txq_lock); 1383 vioif_start_locked(ifp, txq); 1384 mutex_exit(txq->txq_lock); 1385 } 1386 1387 static inline int 1388 vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) 1389 { 1390 struct vioif_softc *sc = ifp->if_softc; 1391 u_int cpuid = cpu_index(curcpu()); 1392 1393 return cpuid % sc->sc_act_nvq_pairs; 1394 } 1395 1396 static void 1397 vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq) 1398 { 1399 1400 vioif_send_common_locked(ifp, txq, true); 1401 } 1402 1403 static int 1404 vioif_transmit(struct ifnet *ifp, struct mbuf *m) 1405 { 1406 struct vioif_softc *sc = ifp->if_softc; 1407 struct vioif_txqueue *txq; 1408 int qid; 1409 1410 qid = vioif_select_txqueue(ifp, m); 1411 txq = &sc->sc_txq[qid]; 1412 1413 if (__predict_false(!pcq_put(txq->txq_intrq, m))) { 1414 m_freem(m); 1415 return ENOBUFS; 1416 } 1417 1418 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1419 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1420 if (m->m_flags & M_MCAST) 1421 if_statinc_ref(nsr, if_omcasts); 1422 IF_STAT_PUTREF(ifp); 1423 1424 if (mutex_tryenter(txq->txq_lock)) { 1425 vioif_transmit_locked(ifp, txq); 1426 mutex_exit(txq->txq_lock); 1427 } 1428 1429 return 0; 1430 } 1431 1432 static void 1433 vioif_deferred_transmit(void *arg) 1434 { 1435 struct vioif_txqueue *txq = arg; 1436 struct virtio_softc *vsc = txq->txq_vq->vq_owner; 1437 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1438 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1439 1440 mutex_enter(txq->txq_lock); 1441 vioif_send_common_locked(ifp, txq, true); 1442 mutex_exit(txq->txq_lock); 1443 } 1444 1445 static int 1446 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 1447 { 1448 int s, r; 1449 1450 s = splnet(); 1451 1452 r = ether_ioctl(ifp, cmd, data); 1453 if ((r == 0 && cmd == SIOCSIFFLAGS) || 1454 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) { 1455 if (ifp->if_flags & IFF_RUNNING) 1456 r = vioif_rx_filter(ifp->if_softc); 1457 else 1458 r = 0; 1459 } 1460 1461 splx(s); 1462 1463 return r; 1464 } 1465 1466 void 1467 vioif_watchdog(struct ifnet *ifp) 1468 { 1469 struct vioif_softc *sc = ifp->if_softc; 1470 int i; 1471 1472 if (ifp->if_flags & IFF_RUNNING) { 1473 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 1474 vioif_tx_queue_clear(&sc->sc_txq[i]); 1475 } 1476 } 1477 } 1478 1479 /* 1480 * Receive implementation 1481 */ 1482 /* allocate and initialize a mbuf for receive */ 1483 static int 1484 vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i) 1485 { 1486 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; 1487 struct mbuf *m; 1488 int r; 1489 1490 MGETHDR(m, M_DONTWAIT, MT_DATA); 1491 if (m == NULL) 1492 return ENOBUFS; 1493 MCLGET(m, M_DONTWAIT); 1494 if ((m->m_flags & M_EXT) == 0) { 1495 m_freem(m); 1496 return ENOBUFS; 1497 } 1498 rxq->rxq_mbufs[i] = m; 1499 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 1500 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 1501 rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT); 1502 if (r) { 1503 m_freem(m); 1504 rxq->rxq_mbufs[i] = NULL; 1505 return r; 1506 } 1507 1508 return 0; 1509 } 1510 1511 /* free a mbuf for receive */ 1512 static void 1513 vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i) 1514 { 1515 struct virtio_softc *vsc = rxq->rxq_vq->vq_owner; 1516 1517 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]); 1518 m_freem(rxq->rxq_mbufs[i]); 1519 rxq->rxq_mbufs[i] = NULL; 1520 } 1521 1522 /* add mbufs for all the empty receive slots */ 1523 static void 1524 vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq) 1525 { 1526 struct virtqueue *vq = rxq->rxq_vq; 1527 struct virtio_softc *vsc = vq->vq_owner; 1528 int i, r, ndone = 0; 1529 1530 KASSERT(mutex_owned(rxq->rxq_lock)); 1531 1532 if (rxq->rxq_stopping) 1533 return; 1534 1535 for (i = 0; i < vq->vq_num; i++) { 1536 int slot; 1537 r = virtio_enqueue_prep(vsc, vq, &slot); 1538 if (r == EAGAIN) 1539 break; 1540 if (r != 0) 1541 panic("enqueue_prep for rx buffers"); 1542 if (rxq->rxq_mbufs[slot] == NULL) { 1543 r = vioif_add_rx_mbuf(rxq, slot); 1544 if (r != 0) { 1545 rxq->rxq_mbuf_add_failed.ev_count++; 1546 break; 1547 } 1548 } 1549 r = virtio_enqueue_reserve(vsc, vq, slot, 1550 rxq->rxq_dmamaps[slot]->dm_nsegs + 1); 1551 if (r != 0) { 1552 vioif_free_rx_mbuf(rxq, slot); 1553 break; 1554 } 1555 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 1556 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); 1557 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 1558 0, MCLBYTES, BUS_DMASYNC_PREREAD); 1559 virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot], 1560 false); 1561 virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false); 1562 virtio_enqueue_commit(vsc, vq, slot, false); 1563 ndone++; 1564 } 1565 if (ndone > 0) 1566 virtio_enqueue_commit(vsc, vq, -1, true); 1567 } 1568 1569 static void 1570 vioif_rx_queue_clear(struct vioif_rxqueue *rxq) 1571 { 1572 struct virtqueue *vq = rxq->rxq_vq; 1573 struct virtio_softc *vsc = vq->vq_owner; 1574 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1575 u_int limit = UINT_MAX; 1576 bool more; 1577 1578 KASSERT(rxq->rxq_stopping); 1579 1580 mutex_enter(rxq->rxq_lock); 1581 for (;;) { 1582 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1583 if (more == false) 1584 break; 1585 } 1586 mutex_exit(rxq->rxq_lock); 1587 } 1588 1589 /* dequeue received packets */ 1590 static bool 1591 vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1592 struct vioif_rxqueue *rxq, u_int limit) 1593 { 1594 struct virtqueue *vq = rxq->rxq_vq; 1595 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1596 struct mbuf *m; 1597 int slot, len; 1598 bool more = false, dequeued = false; 1599 1600 KASSERT(mutex_owned(rxq->rxq_lock)); 1601 1602 if (virtio_vq_is_enqueued(vsc, vq) == false) 1603 return false; 1604 1605 for (;;) { 1606 if (limit-- == 0) { 1607 more = true; 1608 break; 1609 } 1610 1611 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1612 break; 1613 1614 dequeued = true; 1615 1616 len -= sizeof(struct virtio_net_hdr); 1617 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot], 1618 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD); 1619 bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot], 1620 0, MCLBYTES, BUS_DMASYNC_POSTREAD); 1621 m = rxq->rxq_mbufs[slot]; 1622 KASSERT(m != NULL); 1623 bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]); 1624 rxq->rxq_mbufs[slot] = NULL; 1625 virtio_dequeue_commit(vsc, vq, slot); 1626 m_set_rcvif(m, ifp); 1627 m->m_len = m->m_pkthdr.len = len; 1628 1629 mutex_exit(rxq->rxq_lock); 1630 if_percpuq_enqueue(ifp->if_percpuq, m); 1631 mutex_enter(rxq->rxq_lock); 1632 1633 if (rxq->rxq_stopping) 1634 break; 1635 } 1636 1637 if (dequeued) 1638 vioif_populate_rx_mbufs_locked(rxq); 1639 1640 return more; 1641 } 1642 1643 /* rx interrupt; call _dequeue above and schedule a softint */ 1644 static int 1645 vioif_rx_intr(void *arg) 1646 { 1647 struct vioif_rxqueue *rxq = arg; 1648 struct virtqueue *vq = rxq->rxq_vq; 1649 struct virtio_softc *vsc = vq->vq_owner; 1650 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1651 u_int limit; 1652 bool more; 1653 1654 limit = sc->sc_rx_intr_process_limit; 1655 1656 if (atomic_load_relaxed(&rxq->rxq_active) == true) 1657 return 1; 1658 1659 mutex_enter(rxq->rxq_lock); 1660 1661 if (!rxq->rxq_stopping) { 1662 rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl; 1663 1664 virtio_stop_vq_intr(vsc, vq); 1665 atomic_store_relaxed(&rxq->rxq_active, true); 1666 1667 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1668 if (more) { 1669 vioif_rx_sched_handle(sc, rxq); 1670 } else { 1671 atomic_store_relaxed(&rxq->rxq_active, false); 1672 virtio_start_vq_intr(vsc, vq); 1673 } 1674 } 1675 1676 mutex_exit(rxq->rxq_lock); 1677 return 1; 1678 } 1679 1680 static void 1681 vioif_rx_handle(void *xrxq) 1682 { 1683 struct vioif_rxqueue *rxq = xrxq; 1684 struct virtqueue *vq = rxq->rxq_vq; 1685 struct virtio_softc *vsc = vq->vq_owner; 1686 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1687 u_int limit; 1688 bool more; 1689 1690 limit = sc->sc_rx_process_limit; 1691 1692 mutex_enter(rxq->rxq_lock); 1693 1694 if (!rxq->rxq_stopping) { 1695 more = vioif_rx_deq_locked(sc, vsc, rxq, limit); 1696 if (more) { 1697 vioif_rx_sched_handle(sc, rxq); 1698 } else { 1699 atomic_store_relaxed(&rxq->rxq_active, false); 1700 virtio_start_vq_intr(vsc, rxq->rxq_vq); 1701 } 1702 } 1703 1704 mutex_exit(rxq->rxq_lock); 1705 } 1706 1707 static void 1708 vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq) 1709 { 1710 1711 if (rxq->rxq_workqueue) 1712 vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work); 1713 else 1714 softint_schedule(rxq->rxq_handle_si); 1715 } 1716 1717 /* free all the mbufs; called from if_stop(disable) */ 1718 static void 1719 vioif_rx_drain(struct vioif_rxqueue *rxq) 1720 { 1721 struct virtqueue *vq = rxq->rxq_vq; 1722 int i; 1723 1724 for (i = 0; i < vq->vq_num; i++) { 1725 if (rxq->rxq_mbufs[i] == NULL) 1726 continue; 1727 vioif_free_rx_mbuf(rxq, i); 1728 } 1729 } 1730 1731 /* 1732 * Transmition implementation 1733 */ 1734 /* actual transmission is done in if_start */ 1735 /* tx interrupt; dequeue and free mbufs */ 1736 /* 1737 * tx interrupt is actually disabled; this should be called upon 1738 * tx vq full and watchdog 1739 */ 1740 1741 static int 1742 vioif_tx_intr(void *arg) 1743 { 1744 struct vioif_txqueue *txq = arg; 1745 struct virtqueue *vq = txq->txq_vq; 1746 struct virtio_softc *vsc = vq->vq_owner; 1747 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1748 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1749 bool more; 1750 u_int limit; 1751 1752 limit = sc->sc_tx_intr_process_limit; 1753 1754 if (atomic_load_relaxed(&txq->txq_active) == true) 1755 return 1; 1756 1757 mutex_enter(txq->txq_lock); 1758 1759 if (!txq->txq_stopping) { 1760 txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl; 1761 1762 virtio_stop_vq_intr(vsc, vq); 1763 atomic_store_relaxed(&txq->txq_active, true); 1764 1765 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1766 if (more) { 1767 vioif_tx_sched_handle(sc, txq); 1768 } else { 1769 atomic_store_relaxed(&txq->txq_active, false); 1770 1771 /* for ALTQ */ 1772 if (txq == &sc->sc_txq[0]) { 1773 if_schedule_deferred_start(ifp); 1774 ifp->if_flags &= ~IFF_OACTIVE; 1775 } 1776 softint_schedule(txq->txq_deferred_transmit); 1777 1778 virtio_start_vq_intr(vsc, vq); 1779 } 1780 } 1781 1782 mutex_exit(txq->txq_lock); 1783 1784 return 1; 1785 } 1786 1787 static void 1788 vioif_tx_handle(void *xtxq) 1789 { 1790 struct vioif_txqueue *txq = xtxq; 1791 struct virtqueue *vq = txq->txq_vq; 1792 struct virtio_softc *vsc = vq->vq_owner; 1793 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1794 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1795 u_int limit; 1796 bool more; 1797 1798 limit = sc->sc_tx_process_limit; 1799 1800 mutex_enter(txq->txq_lock); 1801 1802 if (!txq->txq_stopping) { 1803 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1804 if (more) { 1805 vioif_tx_sched_handle(sc, txq); 1806 } else { 1807 atomic_store_relaxed(&txq->txq_active, false); 1808 1809 /* for ALTQ */ 1810 if (txq == &sc->sc_txq[0]) { 1811 if_schedule_deferred_start(ifp); 1812 ifp->if_flags &= ~IFF_OACTIVE; 1813 } 1814 softint_schedule(txq->txq_deferred_transmit); 1815 1816 virtio_start_vq_intr(vsc, txq->txq_vq); 1817 } 1818 } 1819 1820 mutex_exit(txq->txq_lock); 1821 } 1822 1823 static void 1824 vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq) 1825 { 1826 1827 if (txq->txq_workqueue) 1828 vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work); 1829 else 1830 softint_schedule(txq->txq_handle_si); 1831 } 1832 1833 static void 1834 vioif_tx_queue_clear(struct vioif_txqueue *txq) 1835 { 1836 struct virtqueue *vq = txq->txq_vq; 1837 struct virtio_softc *vsc = vq->vq_owner; 1838 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1839 u_int limit = UINT_MAX; 1840 bool more; 1841 1842 mutex_enter(txq->txq_lock); 1843 for (;;) { 1844 more = vioif_tx_deq_locked(sc, vsc, txq, limit); 1845 if (more == false) 1846 break; 1847 } 1848 mutex_exit(txq->txq_lock); 1849 } 1850 1851 static bool 1852 vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, 1853 struct vioif_txqueue *txq, u_int limit) 1854 { 1855 struct virtqueue *vq = txq->txq_vq; 1856 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1857 struct mbuf *m; 1858 int slot, len; 1859 bool more = false; 1860 1861 KASSERT(mutex_owned(txq->txq_lock)); 1862 1863 if (virtio_vq_is_enqueued(vsc, vq) == false) 1864 return false; 1865 1866 for (;;) { 1867 if (limit-- == 0) { 1868 more = true; 1869 break; 1870 } 1871 1872 if (virtio_dequeue(vsc, vq, &slot, &len) != 0) 1873 break; 1874 1875 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot], 1876 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE); 1877 bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot], 1878 0, txq->txq_dmamaps[slot]->dm_mapsize, 1879 BUS_DMASYNC_POSTWRITE); 1880 m = txq->txq_mbufs[slot]; 1881 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]); 1882 txq->txq_mbufs[slot] = NULL; 1883 virtio_dequeue_commit(vsc, vq, slot); 1884 if_statinc(ifp, if_opackets); 1885 m_freem(m); 1886 } 1887 1888 return more; 1889 } 1890 1891 /* free all the mbufs already put on vq; called from if_stop(disable) */ 1892 static void 1893 vioif_tx_drain(struct vioif_txqueue *txq) 1894 { 1895 struct virtqueue *vq = txq->txq_vq; 1896 struct virtio_softc *vsc = vq->vq_owner; 1897 int i; 1898 1899 KASSERT(txq->txq_stopping); 1900 1901 for (i = 0; i < vq->vq_num; i++) { 1902 if (txq->txq_mbufs[i] == NULL) 1903 continue; 1904 bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]); 1905 m_freem(txq->txq_mbufs[i]); 1906 txq->txq_mbufs[i] = NULL; 1907 } 1908 } 1909 1910 /* 1911 * Control vq 1912 */ 1913 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1914 static void 1915 vioif_ctrl_acquire(struct vioif_softc *sc) 1916 { 1917 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1918 1919 mutex_enter(&ctrlq->ctrlq_wait_lock); 1920 while (ctrlq->ctrlq_inuse != FREE) 1921 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 1922 ctrlq->ctrlq_inuse = INUSE; 1923 ctrlq->ctrlq_owner = curlwp; 1924 mutex_exit(&ctrlq->ctrlq_wait_lock); 1925 } 1926 1927 static void 1928 vioif_ctrl_release(struct vioif_softc *sc) 1929 { 1930 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1931 1932 KASSERT(ctrlq->ctrlq_inuse != FREE); 1933 KASSERT(ctrlq->ctrlq_owner == curlwp); 1934 1935 mutex_enter(&ctrlq->ctrlq_wait_lock); 1936 ctrlq->ctrlq_inuse = FREE; 1937 ctrlq->ctrlq_owner = NULL; 1938 cv_signal(&ctrlq->ctrlq_wait); 1939 mutex_exit(&ctrlq->ctrlq_wait_lock); 1940 } 1941 1942 static int 1943 vioif_ctrl_load_cmdspec(struct vioif_softc *sc, 1944 struct vioif_ctrl_cmdspec *specs, int nspecs) 1945 { 1946 struct virtio_softc *vsc = sc->sc_virtio; 1947 int i, r, loaded; 1948 1949 loaded = 0; 1950 for (i = 0; i < nspecs; i++) { 1951 r = bus_dmamap_load(virtio_dmat(vsc), 1952 specs[i].dmamap, specs[i].buf, specs[i].bufsize, 1953 NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1954 if (r) { 1955 sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++; 1956 goto err; 1957 } 1958 loaded++; 1959 1960 } 1961 1962 return r; 1963 1964 err: 1965 for (i = 0; i < loaded; i++) { 1966 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 1967 } 1968 1969 return r; 1970 } 1971 1972 static void 1973 vioif_ctrl_unload_cmdspec(struct vioif_softc *sc, 1974 struct vioif_ctrl_cmdspec *specs, int nspecs) 1975 { 1976 struct virtio_softc *vsc = sc->sc_virtio; 1977 int i; 1978 1979 for (i = 0; i < nspecs; i++) { 1980 bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap); 1981 } 1982 } 1983 1984 static int 1985 vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd, 1986 struct vioif_ctrl_cmdspec *specs, int nspecs) 1987 { 1988 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 1989 struct virtqueue *vq = ctrlq->ctrlq_vq; 1990 struct virtio_softc *vsc = sc->sc_virtio; 1991 int i, r, slot; 1992 1993 ctrlq->ctrlq_cmd->class = class; 1994 ctrlq->ctrlq_cmd->command = cmd; 1995 1996 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 1997 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE); 1998 for (i = 0; i < nspecs; i++) { 1999 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 2000 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE); 2001 } 2002 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 2003 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD); 2004 2005 r = virtio_enqueue_prep(vsc, vq, &slot); 2006 if (r != 0) 2007 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2008 r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2); 2009 if (r != 0) 2010 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 2011 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true); 2012 for (i = 0; i < nspecs; i++) { 2013 virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true); 2014 } 2015 virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false); 2016 virtio_enqueue_commit(vsc, vq, slot, true); 2017 2018 /* wait for done */ 2019 mutex_enter(&ctrlq->ctrlq_wait_lock); 2020 while (ctrlq->ctrlq_inuse != DONE) 2021 cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock); 2022 mutex_exit(&ctrlq->ctrlq_wait_lock); 2023 /* already dequeueued */ 2024 2025 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0, 2026 sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE); 2027 for (i = 0; i < nspecs; i++) { 2028 bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0, 2029 specs[i].bufsize, BUS_DMASYNC_POSTWRITE); 2030 } 2031 bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0, 2032 sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD); 2033 2034 if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK) 2035 r = 0; 2036 else { 2037 device_printf(sc->sc_dev, "failed setting rx mode\n"); 2038 sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++; 2039 r = EIO; 2040 } 2041 2042 return r; 2043 } 2044 2045 static int 2046 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 2047 { 2048 struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx; 2049 struct vioif_ctrl_cmdspec specs[1]; 2050 int r; 2051 2052 if (!sc->sc_has_ctrl) 2053 return ENOTSUP; 2054 2055 vioif_ctrl_acquire(sc); 2056 2057 rx->onoff = onoff; 2058 specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap; 2059 specs[0].buf = rx; 2060 specs[0].bufsize = sizeof(*rx); 2061 2062 r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd, 2063 specs, __arraycount(specs)); 2064 2065 vioif_ctrl_release(sc); 2066 return r; 2067 } 2068 2069 static int 2070 vioif_set_promisc(struct vioif_softc *sc, bool onoff) 2071 { 2072 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 2073 } 2074 2075 static int 2076 vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 2077 { 2078 return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 2079 } 2080 2081 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 2082 static int 2083 vioif_set_rx_filter(struct vioif_softc *sc) 2084 { 2085 /* filter already set in ctrlq->ctrlq_mac_tbl */ 2086 struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; 2087 struct vioif_ctrl_cmdspec specs[2]; 2088 int nspecs = __arraycount(specs); 2089 int r; 2090 2091 mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; 2092 mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; 2093 2094 if (!sc->sc_has_ctrl) 2095 return ENOTSUP; 2096 2097 vioif_ctrl_acquire(sc); 2098 2099 specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; 2100 specs[0].buf = mac_tbl_uc; 2101 specs[0].bufsize = sizeof(*mac_tbl_uc) 2102 + (ETHER_ADDR_LEN * mac_tbl_uc->nentries); 2103 2104 specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; 2105 specs[1].buf = mac_tbl_mc; 2106 specs[1].bufsize = sizeof(*mac_tbl_mc) 2107 + (ETHER_ADDR_LEN * mac_tbl_mc->nentries); 2108 2109 r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); 2110 if (r != 0) 2111 goto out; 2112 2113 r = vioif_ctrl_send_command(sc, 2114 VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, 2115 specs, nspecs); 2116 2117 vioif_ctrl_unload_cmdspec(sc, specs, nspecs); 2118 2119 out: 2120 vioif_ctrl_release(sc); 2121 2122 return r; 2123 } 2124 2125 static int 2126 vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) 2127 { 2128 struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; 2129 struct vioif_ctrl_cmdspec specs[1]; 2130 int r; 2131 2132 if (!sc->sc_has_ctrl) 2133 return ENOTSUP; 2134 2135 if (nvq_pairs <= 1) 2136 return EINVAL; 2137 2138 vioif_ctrl_acquire(sc); 2139 2140 mq->virtqueue_pairs = nvq_pairs; 2141 specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; 2142 specs[0].buf = mq; 2143 specs[0].bufsize = sizeof(*mq); 2144 2145 r = vioif_ctrl_send_command(sc, 2146 VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, 2147 specs, __arraycount(specs)); 2148 2149 vioif_ctrl_release(sc); 2150 2151 return r; 2152 } 2153 2154 /* ctrl vq interrupt; wake up the command issuer */ 2155 static int 2156 vioif_ctrl_intr(void *arg) 2157 { 2158 struct vioif_ctrlqueue *ctrlq = arg; 2159 struct virtqueue *vq = ctrlq->ctrlq_vq; 2160 struct virtio_softc *vsc = vq->vq_owner; 2161 int r, slot; 2162 2163 if (virtio_vq_is_enqueued(vsc, vq) == false) 2164 return 0; 2165 2166 r = virtio_dequeue(vsc, vq, &slot, NULL); 2167 if (r == ENOENT) 2168 return 0; 2169 virtio_dequeue_commit(vsc, vq, slot); 2170 2171 mutex_enter(&ctrlq->ctrlq_wait_lock); 2172 ctrlq->ctrlq_inuse = DONE; 2173 cv_signal(&ctrlq->ctrlq_wait); 2174 mutex_exit(&ctrlq->ctrlq_wait_lock); 2175 2176 return 1; 2177 } 2178 2179 /* 2180 * If IFF_PROMISC requested, set promiscuous 2181 * If multicast filter small enough (<=MAXENTRIES) set rx filter 2182 * If large multicast filter exist use ALLMULTI 2183 */ 2184 /* 2185 * If setting rx filter fails fall back to ALLMULTI 2186 * If ALLMULTI fails fall back to PROMISC 2187 */ 2188 static int 2189 vioif_rx_filter(struct vioif_softc *sc) 2190 { 2191 struct ethercom *ec = &sc->sc_ethercom; 2192 struct ifnet *ifp = &ec->ec_if; 2193 struct ether_multi *enm; 2194 struct ether_multistep step; 2195 struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; 2196 int nentries; 2197 int promisc = 0, allmulti = 0, rxfilter = 0; 2198 int r; 2199 2200 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */ 2201 ifp->if_flags |= IFF_PROMISC; 2202 return 0; 2203 } 2204 2205 if (ifp->if_flags & IFF_PROMISC) { 2206 promisc = 1; 2207 goto set; 2208 } 2209 2210 nentries = -1; 2211 ETHER_LOCK(ec); 2212 ETHER_FIRST_MULTI(step, ec, enm); 2213 while (nentries++, enm != NULL) { 2214 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 2215 allmulti = 1; 2216 goto set_unlock; 2217 } 2218 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) { 2219 allmulti = 1; 2220 goto set_unlock; 2221 } 2222 memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries], 2223 enm->enm_addrlo, ETHER_ADDR_LEN); 2224 ETHER_NEXT_MULTI(step, enm); 2225 } 2226 rxfilter = 1; 2227 2228 set_unlock: 2229 ETHER_UNLOCK(ec); 2230 2231 set: 2232 if (rxfilter) { 2233 ctrlq->ctrlq_mac_tbl_uc->nentries = 0; 2234 ctrlq->ctrlq_mac_tbl_mc->nentries = nentries; 2235 r = vioif_set_rx_filter(sc); 2236 if (r != 0) { 2237 rxfilter = 0; 2238 allmulti = 1; /* fallback */ 2239 } 2240 } else { 2241 /* remove rx filter */ 2242 ctrlq->ctrlq_mac_tbl_uc->nentries = 0; 2243 ctrlq->ctrlq_mac_tbl_mc->nentries = 0; 2244 r = vioif_set_rx_filter(sc); 2245 /* what to do on failure? */ 2246 } 2247 if (allmulti) { 2248 r = vioif_set_allmulti(sc, true); 2249 if (r != 0) { 2250 allmulti = 0; 2251 promisc = 1; /* fallback */ 2252 } 2253 } else { 2254 r = vioif_set_allmulti(sc, false); 2255 /* what to do on failure? */ 2256 } 2257 if (promisc) { 2258 r = vioif_set_promisc(sc, true); 2259 } else { 2260 r = vioif_set_promisc(sc, false); 2261 } 2262 2263 return r; 2264 } 2265 2266 static bool 2267 vioif_is_link_up(struct vioif_softc *sc) 2268 { 2269 struct virtio_softc *vsc = sc->sc_virtio; 2270 uint16_t status; 2271 2272 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 2273 status = virtio_read_device_config_2(vsc, 2274 VIRTIO_NET_CONFIG_STATUS); 2275 else 2276 status = VIRTIO_NET_S_LINK_UP; 2277 2278 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 2279 } 2280 2281 /* change link status */ 2282 static void 2283 vioif_update_link_status(struct vioif_softc *sc) 2284 { 2285 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2286 struct vioif_txqueue *txq; 2287 bool active, changed; 2288 int link, i; 2289 2290 mutex_enter(&sc->sc_lock); 2291 2292 active = vioif_is_link_up(sc); 2293 changed = false; 2294 2295 if (active) { 2296 if (!sc->sc_link_active) 2297 changed = true; 2298 2299 link = LINK_STATE_UP; 2300 sc->sc_link_active = true; 2301 } else { 2302 if (sc->sc_link_active) 2303 changed = true; 2304 2305 link = LINK_STATE_DOWN; 2306 sc->sc_link_active = false; 2307 } 2308 2309 if (changed) { 2310 for (i = 0; i < sc->sc_act_nvq_pairs; i++) { 2311 txq = &sc->sc_txq[i]; 2312 2313 mutex_enter(txq->txq_lock); 2314 txq->txq_link_active = sc->sc_link_active; 2315 mutex_exit(txq->txq_lock); 2316 } 2317 2318 if_link_state_change(ifp, link); 2319 } 2320 2321 mutex_exit(&sc->sc_lock); 2322 } 2323 2324 static int 2325 vioif_config_change(struct virtio_softc *vsc) 2326 { 2327 struct vioif_softc *sc = device_private(virtio_child(vsc)); 2328 2329 softint_schedule(sc->sc_ctl_softint); 2330 return 0; 2331 } 2332 2333 static void 2334 vioif_ctl_softint(void *arg) 2335 { 2336 struct vioif_softc *sc = arg; 2337 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 2338 2339 vioif_update_link_status(sc); 2340 vioif_start(ifp); 2341 } 2342 2343 static struct workqueue * 2344 vioif_workq_create(const char *name, pri_t prio, int ipl, int flags) 2345 { 2346 struct workqueue *wq; 2347 int error; 2348 2349 error = workqueue_create(&wq, name, vioif_workq_work, NULL, 2350 prio, ipl, flags); 2351 2352 if (error) 2353 return NULL; 2354 2355 return wq; 2356 } 2357 2358 static void 2359 vioif_workq_destroy(struct workqueue *wq) 2360 { 2361 2362 workqueue_destroy(wq); 2363 } 2364 2365 static void 2366 vioif_workq_work(struct work *wk, void *context) 2367 { 2368 struct vioif_work *work; 2369 2370 work = container_of(wk, struct vioif_work, cookie); 2371 2372 atomic_store_relaxed(&work->added, 0); 2373 work->func(work->arg); 2374 } 2375 2376 static void 2377 vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) 2378 { 2379 2380 memset(work, 0, sizeof(*work)); 2381 work->func = func; 2382 work->arg = arg; 2383 } 2384 2385 static void 2386 vioif_work_add(struct workqueue *wq, struct vioif_work *work) 2387 { 2388 2389 if (atomic_load_relaxed(&work->added) != 0) 2390 return; 2391 2392 atomic_store_relaxed(&work->added, 1); 2393 kpreempt_disable(); 2394 workqueue_enqueue(wq, &work->cookie, NULL); 2395 kpreempt_enable(); 2396 } 2397 2398 static void 2399 vioif_work_wait(struct workqueue *wq, struct vioif_work *work) 2400 { 2401 2402 workqueue_wait(wq, &work->cookie); 2403 } 2404 2405 static int 2406 vioif_setup_sysctl(struct vioif_softc *sc) 2407 { 2408 const char *devname; 2409 struct sysctllog **log; 2410 const struct sysctlnode *rnode, *rxnode, *txnode; 2411 int error; 2412 2413 log = &sc->sc_sysctllog; 2414 devname = device_xname(sc->sc_dev); 2415 2416 error = sysctl_createv(log, 0, NULL, &rnode, 2417 0, CTLTYPE_NODE, devname, 2418 SYSCTL_DESCR("virtio-net information and settings"), 2419 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 2420 if (error) 2421 goto out; 2422 2423 error = sysctl_createv(log, 0, &rnode, NULL, 2424 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2425 SYSCTL_DESCR("Use workqueue for packet processing"), 2426 NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); 2427 if (error) 2428 goto out; 2429 2430 error = sysctl_createv(log, 0, &rnode, &rxnode, 2431 0, CTLTYPE_NODE, "rx", 2432 SYSCTL_DESCR("virtio-net information and settings for Rx"), 2433 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 2434 if (error) 2435 goto out; 2436 2437 error = sysctl_createv(log, 0, &rxnode, NULL, 2438 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 2439 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), 2440 NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 2441 if (error) 2442 goto out; 2443 2444 error = sysctl_createv(log, 0, &rxnode, NULL, 2445 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 2446 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), 2447 NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); 2448 if (error) 2449 goto out; 2450 2451 error = sysctl_createv(log, 0, &rnode, &txnode, 2452 0, CTLTYPE_NODE, "tx", 2453 SYSCTL_DESCR("virtio-net information and settings for Tx"), 2454 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); 2455 if (error) 2456 goto out; 2457 2458 error = sysctl_createv(log, 0, &txnode, NULL, 2459 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", 2460 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), 2461 NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); 2462 if (error) 2463 goto out; 2464 2465 error = sysctl_createv(log, 0, &txnode, NULL, 2466 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", 2467 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), 2468 NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); 2469 2470 out: 2471 if (error) 2472 sysctl_teardown(log); 2473 2474 return error; 2475 } 2476 2477 static void 2478 vioif_setup_stats(struct vioif_softc *sc) 2479 { 2480 struct vioif_rxqueue *rxq; 2481 struct vioif_txqueue *txq; 2482 int i; 2483 2484 for (i = 0; i < sc->sc_max_nvq_pairs; i++) { 2485 rxq = &sc->sc_rxq[i]; 2486 txq = &sc->sc_txq[i]; 2487 2488 snprintf(txq->txq_evgroup, sizeof(txq->txq_evgroup), "%s-TX%d", 2489 device_xname(sc->sc_dev), i); 2490 evcnt_attach_dynamic(&txq->txq_defrag_failed, EVCNT_TYPE_MISC, 2491 NULL, txq->txq_evgroup, "tx m_defrag() failed"); 2492 evcnt_attach_dynamic(&txq->txq_mbuf_load_failed, EVCNT_TYPE_MISC, 2493 NULL, txq->txq_evgroup, "tx dmamap load failed"); 2494 evcnt_attach_dynamic(&txq->txq_enqueue_reserve_failed, EVCNT_TYPE_MISC, 2495 NULL, txq->txq_evgroup, "virtio_enqueue_reserve failed"); 2496 2497 snprintf(rxq->rxq_evgroup, sizeof(rxq->rxq_evgroup), "%s-RX%d", 2498 device_xname(sc->sc_dev), i); 2499 evcnt_attach_dynamic(&rxq->rxq_mbuf_add_failed, EVCNT_TYPE_MISC, 2500 NULL, rxq->rxq_evgroup, "rx mbuf allocation failed"); 2501 } 2502 2503 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, 2504 NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); 2505 evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, 2506 NULL, device_xname(sc->sc_dev), "control command failed"); 2507 } 2508 2509 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 2510 2511 #ifdef _MODULE 2512 #include "ioconf.c" 2513 #endif 2514 2515 static int 2516 if_vioif_modcmd(modcmd_t cmd, void *opaque) 2517 { 2518 int error = 0; 2519 2520 #ifdef _MODULE 2521 switch (cmd) { 2522 case MODULE_CMD_INIT: 2523 error = config_init_component(cfdriver_ioconf_if_vioif, 2524 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2525 break; 2526 case MODULE_CMD_FINI: 2527 error = config_fini_component(cfdriver_ioconf_if_vioif, 2528 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 2529 break; 2530 default: 2531 error = ENOTTY; 2532 break; 2533 } 2534 #endif 2535 2536 return error; 2537 } 2538