1 /* $NetBSD: if_vmx.c,v 1.17 2024/07/05 04:31:51 rin Exp $ */
2 /* $OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $ */
3
4 /*
5 * Copyright (c) 2013 Tsubai Masanari
6 * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
7 *
8 * Permission to use, copy, modify, and distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 */
20
21 #include <sys/cdefs.h>
22 __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.17 2024/07/05 04:31:51 rin Exp $");
23
24 #ifdef _KERNEL_OPT
25 #include "opt_if_vmx.h"
26 #endif
27
28 #include <sys/param.h>
29 #include <sys/cpu.h>
30 #include <sys/kernel.h>
31 #include <sys/kmem.h>
32 #include <sys/bitops.h>
33 #include <sys/bus.h>
34 #include <sys/device.h>
35 #include <sys/mbuf.h>
36 #include <sys/module.h>
37 #include <sys/sockio.h>
38 #include <sys/pcq.h>
39 #include <sys/workqueue.h>
40 #include <sys/interrupt.h>
41
42 #include <net/bpf.h>
43 #include <net/if.h>
44 #include <net/if_ether.h>
45 #include <net/if_media.h>
46
47 #include <netinet/if_inarp.h>
48 #include <netinet/in_systm.h> /* for <netinet/ip.h> */
49 #include <netinet/in.h> /* for <netinet/ip.h> */
50 #include <netinet/ip.h> /* for struct ip */
51 #include <netinet/ip6.h> /* for struct ip6_hdr */
52 #include <netinet/tcp.h> /* for struct tcphdr */
53 #include <netinet/udp.h> /* for struct udphdr */
54
55 #include <dev/pci/pcivar.h>
56 #include <dev/pci/pcireg.h>
57 #include <dev/pci/pcidevs.h>
58
59 #include <dev/pci/if_vmxreg.h>
60
61 #define VMXNET3_DRIVER_VERSION 0x00010000
62
63 /*
64 * Max descriptors per Tx packet. We must limit the size of the
65 * any TSO packets based on the number of segments.
66 */
67 #define VMXNET3_TX_MAXSEGS 32
68 #define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES)
69
70 /*
71 * Maximum support Tx segments size. The length field in the
72 * Tx descriptor is 14 bits.
73 */
74 #define VMXNET3_TX_MAXSEGSIZE (1 << 14)
75
76 /*
77 * The maximum number of Rx segments we accept.
78 */
79 #define VMXNET3_MAX_RX_SEGS 0 /* no segments */
80
81 /*
82 * Predetermined size of the multicast MACs filter table. If the
83 * number of multicast addresses exceeds this size, then the
84 * ALL_MULTI mode is use instead.
85 */
86 #define VMXNET3_MULTICAST_MAX 32
87
88 /*
89 * Our Tx watchdog timeout.
90 */
91 #define VMXNET3_WATCHDOG_TIMEOUT 5
92
93 /*
94 * Default value for vmx_intr_{rx,tx}_process_limit which is used for
95 * max number of packets to process for interrupt handler
96 */
97 #define VMXNET3_RX_INTR_PROCESS_LIMIT 0U
98 #define VMXNET3_TX_INTR_PROCESS_LIMIT 256
99
100 /*
101 * Default value for vmx_{rx,tx}_process_limit which is used for
102 * max number of packets to process for deferred processing
103 */
104 #define VMXNET3_RX_PROCESS_LIMIT 256
105 #define VMXNET3_TX_PROCESS_LIMIT 256
106
107 #define VMXNET3_WORKQUEUE_PRI PRI_SOFTNET
108
109 /*
110 * IP protocols that we can perform Tx checksum offloading of.
111 */
112 #define VMXNET3_CSUM_OFFLOAD \
113 (M_CSUM_TCPv4 | M_CSUM_UDPv4)
114 #define VMXNET3_CSUM_OFFLOAD_IPV6 \
115 (M_CSUM_TCPv6 | M_CSUM_UDPv6)
116
117 #define VMXNET3_CSUM_ALL_OFFLOAD \
118 (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)
119
120 #define VMXNET3_RXRINGS_PERQ 2
121
122 #define VMXNET3_CORE_LOCK(_sc) mutex_enter((_sc)->vmx_mtx)
123 #define VMXNET3_CORE_UNLOCK(_sc) mutex_exit((_sc)->vmx_mtx)
124 #define VMXNET3_CORE_LOCK_ASSERT(_sc) mutex_owned((_sc)->vmx_mtx)
125
126 #define VMXNET3_RXQ_LOCK(_rxq) mutex_enter((_rxq)->vxrxq_mtx)
127 #define VMXNET3_RXQ_UNLOCK(_rxq) mutex_exit((_rxq)->vxrxq_mtx)
128 #define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \
129 mutex_owned((_rxq)->vxrxq_mtx)
130
131 #define VMXNET3_TXQ_LOCK(_txq) mutex_enter((_txq)->vxtxq_mtx)
132 #define VMXNET3_TXQ_TRYLOCK(_txq) mutex_tryenter((_txq)->vxtxq_mtx)
133 #define VMXNET3_TXQ_UNLOCK(_txq) mutex_exit((_txq)->vxtxq_mtx)
134 #define VMXNET3_TXQ_LOCK_ASSERT(_txq) \
135 mutex_owned((_txq)->vxtxq_mtx)
136
137 struct vmxnet3_dma_alloc {
138 bus_addr_t dma_paddr;
139 void *dma_vaddr;
140 bus_dmamap_t dma_map;
141 bus_size_t dma_size;
142 bus_dma_segment_t dma_segs[1];
143 };
144
145 struct vmxnet3_txbuf {
146 bus_dmamap_t vtxb_dmamap;
147 struct mbuf *vtxb_m;
148 };
149
150 struct vmxnet3_txring {
151 struct vmxnet3_txbuf *vxtxr_txbuf;
152 struct vmxnet3_txdesc *vxtxr_txd;
153 u_int vxtxr_head;
154 u_int vxtxr_next;
155 u_int vxtxr_ndesc;
156 int vxtxr_gen;
157 struct vmxnet3_dma_alloc vxtxr_dma;
158 };
159
160 struct vmxnet3_rxbuf {
161 bus_dmamap_t vrxb_dmamap;
162 struct mbuf *vrxb_m;
163 };
164
165 struct vmxnet3_rxring {
166 struct vmxnet3_rxbuf *vxrxr_rxbuf;
167 struct vmxnet3_rxdesc *vxrxr_rxd;
168 u_int vxrxr_fill;
169 u_int vxrxr_ndesc;
170 int vxrxr_gen;
171 int vxrxr_rid;
172 struct vmxnet3_dma_alloc vxrxr_dma;
173 bus_dmamap_t vxrxr_spare_dmap;
174 };
175
176 struct vmxnet3_comp_ring {
177 union {
178 struct vmxnet3_txcompdesc *txcd;
179 struct vmxnet3_rxcompdesc *rxcd;
180 } vxcr_u;
181 u_int vxcr_next;
182 u_int vxcr_ndesc;
183 int vxcr_gen;
184 struct vmxnet3_dma_alloc vxcr_dma;
185 };
186
187 struct vmxnet3_txq_stats {
188 uint64_t vmtxs_csum;
189 uint64_t vmtxs_tso;
190 uint64_t vmtxs_full;
191 uint64_t vmtxs_offload_failed;
192 };
193
194 struct vmxnet3_txqueue {
195 kmutex_t *vxtxq_mtx;
196 struct vmxnet3_softc *vxtxq_sc;
197 int vxtxq_watchdog;
198 pcq_t *vxtxq_interq;
199 struct vmxnet3_txring vxtxq_cmd_ring;
200 struct vmxnet3_comp_ring vxtxq_comp_ring;
201 struct vmxnet3_txq_stats vxtxq_stats;
202 struct vmxnet3_txq_shared *vxtxq_ts;
203 char vxtxq_name[16];
204
205 void *vxtxq_si;
206
207 struct evcnt vxtxq_intr;
208 struct evcnt vxtxq_defer;
209 struct evcnt vxtxq_deferreq;
210 struct evcnt vxtxq_pcqdrop;
211 struct evcnt vxtxq_transmitdef;
212 struct evcnt vxtxq_watchdogto;
213 struct evcnt vxtxq_defragged;
214 struct evcnt vxtxq_defrag_failed;
215
216 bool vxtxq_stopping;
217 };
218
219
220 struct vmxnet3_rxqueue {
221 kmutex_t *vxrxq_mtx;
222 struct vmxnet3_softc *vxrxq_sc;
223 struct mbuf *vxrxq_mhead;
224 struct mbuf *vxrxq_mtail;
225 struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ];
226 struct vmxnet3_comp_ring vxrxq_comp_ring;
227 struct vmxnet3_rxq_shared *vxrxq_rs;
228 char vxrxq_name[16];
229
230 struct evcnt vxrxq_intr;
231 struct evcnt vxrxq_defer;
232 struct evcnt vxrxq_deferreq;
233 struct evcnt vxrxq_mgetcl_failed;
234 struct evcnt vxrxq_mbuf_load_failed;
235
236 bool vxrxq_stopping;
237 };
238
239 struct vmxnet3_queue {
240 int vxq_id;
241 int vxq_intr_idx;
242
243 struct vmxnet3_txqueue vxq_txqueue;
244 struct vmxnet3_rxqueue vxq_rxqueue;
245
246 void *vxq_si;
247 bool vxq_workqueue;
248 bool vxq_wq_enqueued;
249 struct work vxq_wq_cookie;
250 };
251
252 struct vmxnet3_softc {
253 device_t vmx_dev;
254 struct ethercom vmx_ethercom;
255 struct ifmedia vmx_media;
256 struct vmxnet3_driver_shared *vmx_ds;
257 int vmx_flags;
258 #define VMXNET3_FLAG_NO_MSIX (1 << 0)
259 #define VMXNET3_FLAG_RSS (1 << 1)
260 #define VMXNET3_FLAG_ATTACHED (1 << 2)
261
262 struct vmxnet3_queue *vmx_queue;
263
264 struct pci_attach_args *vmx_pa;
265 pci_chipset_tag_t vmx_pc;
266
267 bus_space_tag_t vmx_iot0;
268 bus_space_tag_t vmx_iot1;
269 bus_space_handle_t vmx_ioh0;
270 bus_space_handle_t vmx_ioh1;
271 bus_size_t vmx_ios0;
272 bus_size_t vmx_ios1;
273 bus_dma_tag_t vmx_dmat;
274
275 int vmx_link_active;
276 int vmx_ntxqueues;
277 int vmx_nrxqueues;
278 int vmx_ntxdescs;
279 int vmx_nrxdescs;
280 int vmx_max_rxsegs;
281
282 struct evcnt vmx_event_intr;
283 struct evcnt vmx_event_link;
284 struct evcnt vmx_event_txqerror;
285 struct evcnt vmx_event_rxqerror;
286 struct evcnt vmx_event_dic;
287 struct evcnt vmx_event_debug;
288
289 int vmx_intr_type;
290 int vmx_intr_mask_mode;
291 int vmx_event_intr_idx;
292 int vmx_nintrs;
293 pci_intr_handle_t *vmx_intrs; /* legacy use vmx_intrs[0] */
294 void *vmx_ihs[VMXNET3_MAX_INTRS];
295
296 kmutex_t *vmx_mtx;
297
298 int vmx_if_flags;
299 bool vmx_promisc;
300 bool vmx_mcastactive;
301 uint8_t *vmx_mcast;
302 void *vmx_qs;
303 struct vmxnet3_rss_shared *vmx_rss;
304 callout_t vmx_tick;
305 struct vmxnet3_dma_alloc vmx_ds_dma;
306 struct vmxnet3_dma_alloc vmx_qs_dma;
307 struct vmxnet3_dma_alloc vmx_mcast_dma;
308 struct vmxnet3_dma_alloc vmx_rss_dma;
309 int vmx_max_ntxqueues;
310 int vmx_max_nrxqueues;
311 uint8_t vmx_lladdr[ETHER_ADDR_LEN];
312
313 u_int vmx_rx_intr_process_limit;
314 u_int vmx_tx_intr_process_limit;
315 u_int vmx_rx_process_limit;
316 u_int vmx_tx_process_limit;
317 struct sysctllog *vmx_sysctllog;
318
319 bool vmx_txrx_workqueue;
320 struct workqueue *vmx_queue_wq;
321
322 struct workqueue *vmx_reset_wq;
323 struct work vmx_reset_work;
324 bool vmx_reset_pending;
325 };
326
327 #define VMXNET3_STAT
328
329 #ifdef VMXNET3_STAT
330 struct {
331 u_int txhead;
332 u_int txdone;
333 u_int maxtxlen;
334 u_int rxdone;
335 u_int rxfill;
336 u_int intr;
337 } vmxstat;
338 #endif
339
340 typedef enum {
341 VMXNET3_BARRIER_RD,
342 VMXNET3_BARRIER_WR,
343 } vmxnet3_barrier_t;
344
345 #define JUMBO_LEN (MCLBYTES - ETHER_ALIGN) /* XXX */
346 #define DMAADDR(map) ((map)->dm_segs[0].ds_addr)
347
348 #define vtophys(va) 0 /* XXX ok? */
349
350 static int vmxnet3_match(device_t, cfdata_t, void *);
351 static void vmxnet3_attach(device_t, device_t, void *);
352 static int vmxnet3_detach(device_t, int);
353
354 static int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *);
355 static void vmxnet3_free_pci_resources(struct vmxnet3_softc *);
356 static int vmxnet3_check_version(struct vmxnet3_softc *);
357 static void vmxnet3_check_multiqueue(struct vmxnet3_softc *);
358
359 static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
360 static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
361 static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
362 static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
363 static void vmxnet3_free_interrupts(struct vmxnet3_softc *);
364
365 static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
366 static int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *);
367 static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
368 static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *);
369 static int vmxnet3_setup_interrupts(struct vmxnet3_softc *);
370 static int vmxnet3_setup_sysctl(struct vmxnet3_softc *);
371
372 static int vmxnet3_setup_stats(struct vmxnet3_softc *);
373 static void vmxnet3_teardown_stats(struct vmxnet3_softc *);
374
375 static int vmxnet3_init_rxq(struct vmxnet3_softc *, int);
376 static int vmxnet3_init_txq(struct vmxnet3_softc *, int);
377 static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
378 static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
379 static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
380 static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
381
382 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
383 static void vmxnet3_free_shared_data(struct vmxnet3_softc *);
384 static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
385 static void vmxnet3_free_txq_data(struct vmxnet3_softc *);
386 static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
387 static void vmxnet3_free_rxq_data(struct vmxnet3_softc *);
388 static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
389 static void vmxnet3_free_queue_data(struct vmxnet3_softc *);
390 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
391 static void vmxnet3_free_mcast_table(struct vmxnet3_softc *);
392 static void vmxnet3_init_shared_data(struct vmxnet3_softc *);
393 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
394 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
395 static int vmxnet3_alloc_data(struct vmxnet3_softc *);
396 static void vmxnet3_free_data(struct vmxnet3_softc *);
397 static int vmxnet3_setup_interface(struct vmxnet3_softc *);
398
399 static void vmxnet3_evintr(struct vmxnet3_softc *);
400 static bool vmxnet3_txq_eof(struct vmxnet3_txqueue *, u_int);
401 static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxqueue *,
402 struct vmxnet3_rxring *);
403 static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
404 struct vmxnet3_rxring *, int);
405 static void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *);
406 static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
407 static void vmxnet3_rxq_input(struct vmxnet3_rxqueue *,
408 struct vmxnet3_rxcompdesc *, struct mbuf *);
409 static bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *, u_int);
410 static int vmxnet3_legacy_intr(void *);
411 static int vmxnet3_txrxq_intr(void *);
412 static void vmxnet3_handle_queue(void *);
413 static void vmxnet3_handle_queue_work(struct work *, void *);
414 static int vmxnet3_event_intr(void *);
415
416 static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
417 static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
418 static void vmxnet3_stop_locked(struct vmxnet3_softc *);
419 static void vmxnet3_stop_rendezvous(struct vmxnet3_softc *);
420 static void vmxnet3_stop(struct ifnet *, int);
421
422 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
423 static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
424 static int vmxnet3_reinit_queues(struct vmxnet3_softc *);
425 static int vmxnet3_enable_device(struct vmxnet3_softc *);
426 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
427 static int vmxnet3_reinit(struct vmxnet3_softc *);
428
429 static int vmxnet3_init_locked(struct vmxnet3_softc *);
430 static int vmxnet3_init(struct ifnet *);
431
432 static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *);
433 static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t);
434 static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
435 static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
436 static void vmxnet3_start_locked(struct ifnet *);
437 static void vmxnet3_start(struct ifnet *);
438 static void vmxnet3_transmit_locked(struct ifnet *, struct vmxnet3_txqueue *);
439 static int vmxnet3_transmit(struct ifnet *, struct mbuf *);
440 static void vmxnet3_deferred_transmit(void *);
441
442 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *);
443 static int vmxnet3_ioctl(struct ifnet *, u_long, void *);
444 static int vmxnet3_ifflags_cb(struct ethercom *);
445
446 static int vmxnet3_watchdog(struct vmxnet3_txqueue *);
447 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
448 static void vmxnet3_tick(void *);
449 static void vmxnet3_reset_work(struct work *, void *);
450 static void vmxnet3_if_link_status(struct vmxnet3_softc *);
451 static bool vmxnet3_cmd_link_status(struct ifnet *);
452 static void vmxnet3_ifmedia_status(struct ifnet *, struct ifmediareq *);
453 static int vmxnet3_ifmedia_change(struct ifnet *);
454 static void vmxnet3_set_lladdr(struct vmxnet3_softc *);
455 static void vmxnet3_get_lladdr(struct vmxnet3_softc *);
456
457 static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
458 static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
459
460 static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t,
461 struct vmxnet3_dma_alloc *);
462 static void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *);
463
464 CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc),
465 vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0);
466
467 /* round down to the nearest power of 2 */
468 static int
vmxnet3_calc_queue_size(int n)469 vmxnet3_calc_queue_size(int n)
470 {
471
472 if (__predict_false(n <= 0))
473 return 1;
474
475 return (1U << (fls32(n) - 1));
476 }
477
478 static inline void
vmxnet3_write_bar0(struct vmxnet3_softc * sc,bus_size_t r,uint32_t v)479 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
480 {
481
482 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
483 }
484
485 static inline uint32_t
vmxnet3_read_bar1(struct vmxnet3_softc * sc,bus_size_t r)486 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
487 {
488
489 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
490 }
491
492 static inline void
vmxnet3_write_bar1(struct vmxnet3_softc * sc,bus_size_t r,uint32_t v)493 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
494 {
495
496 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
497 }
498
499 static inline void
vmxnet3_write_cmd(struct vmxnet3_softc * sc,uint32_t cmd)500 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
501 {
502
503 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
504 }
505
506 static inline uint32_t
vmxnet3_read_cmd(struct vmxnet3_softc * sc,uint32_t cmd)507 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
508 {
509
510 vmxnet3_write_cmd(sc, cmd);
511 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
512 }
513
514 static inline void
vmxnet3_enable_intr(struct vmxnet3_softc * sc,int irq)515 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
516 {
517 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
518 }
519
520 static inline void
vmxnet3_disable_intr(struct vmxnet3_softc * sc,int irq)521 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
522 {
523 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
524 }
525
526 static inline void
vmxnet3_rxr_increment_fill(struct vmxnet3_rxring * rxr)527 vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr)
528 {
529
530 if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) {
531 rxr->vxrxr_fill = 0;
532 rxr->vxrxr_gen ^= 1;
533 }
534 }
535
536 static inline int
vmxnet3_txring_avail(struct vmxnet3_txring * txr)537 vmxnet3_txring_avail(struct vmxnet3_txring *txr)
538 {
539 int avail = txr->vxtxr_next - txr->vxtxr_head - 1;
540 return (avail < 0 ? (int)txr->vxtxr_ndesc + avail : avail);
541 }
542
543 /*
544 * Since this is a purely paravirtualized device, we do not have
545 * to worry about DMA coherency. But at times, we must make sure
546 * both the compiler and CPU do not reorder memory operations.
547 */
548 static inline void
vmxnet3_barrier(struct vmxnet3_softc * sc,vmxnet3_barrier_t type)549 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
550 {
551
552 switch (type) {
553 case VMXNET3_BARRIER_RD:
554 membar_consumer();
555 break;
556 case VMXNET3_BARRIER_WR:
557 membar_producer();
558 break;
559 default:
560 panic("%s: bad barrier type %d", __func__, type);
561 }
562 }
563
564 static int
vmxnet3_match(device_t parent,cfdata_t match,void * aux)565 vmxnet3_match(device_t parent, cfdata_t match, void *aux)
566 {
567 struct pci_attach_args *pa = (struct pci_attach_args *)aux;
568
569 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE &&
570 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_VMXNET3)
571 return 1;
572
573 return 0;
574 }
575
576 static void
vmxnet3_attach(device_t parent,device_t self,void * aux)577 vmxnet3_attach(device_t parent, device_t self, void *aux)
578 {
579 struct vmxnet3_softc *sc = device_private(self);
580 struct pci_attach_args *pa = aux;
581 pcireg_t preg;
582 int error;
583 int candidate;
584
585 sc->vmx_dev = self;
586 sc->vmx_pa = pa;
587 sc->vmx_pc = pa->pa_pc;
588 if (pci_dma64_available(pa))
589 sc->vmx_dmat = pa->pa_dmat64;
590 else
591 sc->vmx_dmat = pa->pa_dmat;
592
593 pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1);
594
595 preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
596 preg |= PCI_COMMAND_MASTER_ENABLE;
597 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg);
598
599 sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
600 callout_init(&sc->vmx_tick, CALLOUT_MPSAFE);
601
602 candidate = MIN(MIN(VMXNET3_MAX_TX_QUEUES, VMXNET3_MAX_RX_QUEUES),
603 ncpu);
604 sc->vmx_max_ntxqueues = sc->vmx_max_nrxqueues =
605 vmxnet3_calc_queue_size(candidate);
606 sc->vmx_ntxdescs = 512;
607 sc->vmx_nrxdescs = 256;
608 sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
609
610 error = vmxnet3_alloc_pci_resources(sc);
611 if (error)
612 return;
613
614 error = vmxnet3_check_version(sc);
615 if (error)
616 return;
617
618 error = vmxnet3_alloc_rxtx_queues(sc);
619 if (error)
620 return;
621
622 error = vmxnet3_alloc_interrupts(sc);
623 if (error)
624 return;
625
626 vmxnet3_check_multiqueue(sc);
627
628 error = vmxnet3_alloc_data(sc);
629 if (error)
630 return;
631
632 error = vmxnet3_setup_interface(sc);
633 if (error)
634 return;
635
636 error = vmxnet3_setup_interrupts(sc);
637 if (error)
638 return;
639
640 error = vmxnet3_setup_sysctl(sc);
641 if (error)
642 return;
643
644 error = vmxnet3_setup_stats(sc);
645 if (error)
646 return;
647
648 char buf[128];
649 snprintf(buf, sizeof(buf), "%s_reset", device_xname(sc->vmx_dev));
650 error = workqueue_create(&sc->vmx_reset_wq, "%s_reset",
651 vmxnet3_reset_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_SOFTCLOCK,
652 WQ_MPSAFE);
653 if (error) {
654 aprint_error_dev(sc->vmx_dev,
655 "failed to create reset workqueue: %d\n",
656 error);
657 return;
658 }
659
660 sc->vmx_flags |= VMXNET3_FLAG_ATTACHED;
661 }
662
663 static int
vmxnet3_detach(device_t self,int flags)664 vmxnet3_detach(device_t self, int flags)
665 {
666 struct vmxnet3_softc *sc;
667 struct ifnet *ifp;
668
669 sc = device_private(self);
670 ifp = &sc->vmx_ethercom.ec_if;
671
672 if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) {
673 VMXNET3_CORE_LOCK(sc);
674 vmxnet3_stop_locked(sc);
675 callout_halt(&sc->vmx_tick, sc->vmx_mtx);
676 callout_destroy(&sc->vmx_tick);
677 VMXNET3_CORE_UNLOCK(sc);
678
679 ether_ifdetach(ifp);
680 if_detach(ifp);
681 ifmedia_fini(&sc->vmx_media);
682 }
683
684 vmxnet3_teardown_stats(sc);
685 sysctl_teardown(&sc->vmx_sysctllog);
686
687 vmxnet3_free_interrupts(sc);
688
689 vmxnet3_free_data(sc);
690 vmxnet3_free_pci_resources(sc);
691 vmxnet3_free_rxtx_queues(sc);
692
693 if (sc->vmx_mtx)
694 mutex_obj_free(sc->vmx_mtx);
695
696 return (0);
697 }
698
699 static int
vmxnet3_alloc_pci_resources(struct vmxnet3_softc * sc)700 vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc)
701 {
702 struct pci_attach_args *pa = sc->vmx_pa;
703 pcireg_t memtype;
704
705 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0));
706 if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0,
707 NULL, &sc->vmx_ios0)) {
708 aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n");
709 return (ENXIO);
710 }
711 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1));
712 if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1,
713 NULL, &sc->vmx_ios1)) {
714 aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n");
715 return (ENXIO);
716 }
717
718 if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) {
719 sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
720 return (0);
721 }
722
723 return (0);
724 }
725
726 static void
vmxnet3_free_pci_resources(struct vmxnet3_softc * sc)727 vmxnet3_free_pci_resources(struct vmxnet3_softc *sc)
728 {
729
730 if (sc->vmx_ios0) {
731 bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0);
732 sc->vmx_ios0 = 0;
733 }
734
735 if (sc->vmx_ios1) {
736 bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1);
737 sc->vmx_ios1 = 0;
738 }
739 }
740
741 static int
vmxnet3_check_version(struct vmxnet3_softc * sc)742 vmxnet3_check_version(struct vmxnet3_softc *sc)
743 {
744 u_int ver;
745
746 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
747 if ((ver & 0x1) == 0) {
748 aprint_error_dev(sc->vmx_dev,
749 "unsupported hardware version 0x%x\n", ver);
750 return (ENOTSUP);
751 }
752 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
753
754 ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
755 if ((ver & 0x1) == 0) {
756 aprint_error_dev(sc->vmx_dev,
757 "incompatible UPT version 0x%x\n", ver);
758 return (ENOTSUP);
759 }
760 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
761
762 return (0);
763 }
764
765 static void
vmxnet3_check_multiqueue(struct vmxnet3_softc * sc)766 vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
767 {
768
769 if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
770 goto out;
771
772 /* Just use the maximum configured for now. */
773 sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
774 sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
775
776 if (sc->vmx_nrxqueues > 1)
777 sc->vmx_flags |= VMXNET3_FLAG_RSS;
778
779 return;
780
781 out:
782 sc->vmx_ntxqueues = 1;
783 sc->vmx_nrxqueues = 1;
784 }
785
786 static int
vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc * sc)787 vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
788 {
789 int required;
790 struct pci_attach_args *pa = sc->vmx_pa;
791
792 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
793 return (1);
794
795 /* Allocate an additional vector for the events interrupt. */
796 required = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues) + 1;
797
798 if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required)
799 return (1);
800
801 if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) {
802 sc->vmx_nintrs = required;
803 return (0);
804 }
805
806 return (1);
807 }
808
809 static int
vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc * sc)810 vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
811 {
812 int nmsi, required;
813 struct pci_attach_args *pa = sc->vmx_pa;
814
815 required = 1;
816
817 nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag);
818 if (nmsi < required)
819 return (1);
820
821 if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) {
822 sc->vmx_nintrs = required;
823 return (0);
824 }
825
826 return (1);
827 }
828
829 static int
vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc * sc)830 vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
831 {
832
833 if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) {
834 sc->vmx_nintrs = 1;
835 return (0);
836 }
837
838 return (1);
839 }
840
841 static int
vmxnet3_alloc_interrupts(struct vmxnet3_softc * sc)842 vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
843 {
844 u_int config;
845 int error;
846
847 config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
848
849 sc->vmx_intr_type = config & 0x03;
850 sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
851
852 switch (sc->vmx_intr_type) {
853 case VMXNET3_IT_AUTO:
854 sc->vmx_intr_type = VMXNET3_IT_MSIX;
855 /* FALLTHROUGH */
856 case VMXNET3_IT_MSIX:
857 error = vmxnet3_alloc_msix_interrupts(sc);
858 if (error == 0)
859 break;
860 sc->vmx_intr_type = VMXNET3_IT_MSI;
861 /* FALLTHROUGH */
862 case VMXNET3_IT_MSI:
863 error = vmxnet3_alloc_msi_interrupts(sc);
864 if (error == 0)
865 break;
866 sc->vmx_intr_type = VMXNET3_IT_LEGACY;
867 /* FALLTHROUGH */
868 case VMXNET3_IT_LEGACY:
869 error = vmxnet3_alloc_legacy_interrupts(sc);
870 if (error == 0)
871 break;
872 /* FALLTHROUGH */
873 default:
874 sc->vmx_intr_type = -1;
875 aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n");
876 return (ENXIO);
877 }
878
879 return (error);
880 }
881
882 static void
vmxnet3_free_interrupts(struct vmxnet3_softc * sc)883 vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
884 {
885 pci_chipset_tag_t pc = sc->vmx_pc;
886 int i;
887
888 workqueue_destroy(sc->vmx_queue_wq);
889 for (i = 0; i < sc->vmx_ntxqueues; i++) {
890 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i];
891
892 softint_disestablish(vmxq->vxq_si);
893 vmxq->vxq_si = NULL;
894 }
895 for (i = 0; i < sc->vmx_nintrs; i++) {
896 pci_intr_disestablish(pc, sc->vmx_ihs[i]);
897 }
898 pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs);
899 }
900
901 static int
vmxnet3_setup_msix_interrupts(struct vmxnet3_softc * sc)902 vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
903 {
904 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
905 struct vmxnet3_queue *vmxq;
906 pci_intr_handle_t *intr;
907 void **ihs;
908 int intr_idx, i, use_queues, error;
909 kcpuset_t *affinity;
910 const char *intrstr;
911 char intrbuf[PCI_INTRSTR_LEN];
912 char xnamebuf[32];
913
914 intr = sc->vmx_intrs;
915 intr_idx = 0;
916 ihs = sc->vmx_ihs;
917
918 /* See vmxnet3_alloc_msix_interrupts() */
919 use_queues = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues);
920 for (i = 0; i < use_queues; i++, intr++, ihs++, intr_idx++) {
921 snprintf(xnamebuf, 32, "%s: txrx %d", device_xname(sc->vmx_dev), i);
922
923 vmxq = &sc->vmx_queue[i];
924
925 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
926
927 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
928 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
929 vmxnet3_txrxq_intr, vmxq, xnamebuf);
930 if (*ihs == NULL) {
931 aprint_error_dev(sc->vmx_dev,
932 "unable to establish txrx interrupt at %s\n", intrstr);
933 return (-1);
934 }
935 aprint_normal_dev(sc->vmx_dev, "txrx interrupting at %s\n", intrstr);
936
937 kcpuset_create(&affinity, true);
938 kcpuset_set(affinity, intr_idx % ncpu);
939 error = interrupt_distribute(*ihs, affinity, NULL);
940 if (error) {
941 aprint_normal_dev(sc->vmx_dev,
942 "%s cannot be changed affinity, use default CPU\n",
943 intrstr);
944 }
945 kcpuset_destroy(affinity);
946
947 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
948 vmxnet3_handle_queue, vmxq);
949 if (vmxq->vxq_si == NULL) {
950 aprint_error_dev(sc->vmx_dev,
951 "softint_establish for vxq_si failed\n");
952 return (-1);
953 }
954
955 vmxq->vxq_intr_idx = intr_idx;
956 }
957 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(sc->vmx_dev));
958 error = workqueue_create(&sc->vmx_queue_wq, xnamebuf,
959 vmxnet3_handle_queue_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_NET,
960 WQ_PERCPU | WQ_MPSAFE);
961 if (error) {
962 aprint_error_dev(sc->vmx_dev, "workqueue_create failed\n");
963 return (-1);
964 }
965 sc->vmx_txrx_workqueue = false;
966
967 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
968
969 snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev));
970 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
971 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
972 vmxnet3_event_intr, sc, xnamebuf);
973 if (*ihs == NULL) {
974 aprint_error_dev(sc->vmx_dev,
975 "unable to establish event interrupt at %s\n", intrstr);
976 return (-1);
977 }
978 aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr);
979
980 sc->vmx_event_intr_idx = intr_idx;
981
982 return (0);
983 }
984
985 static int
vmxnet3_setup_msi_interrupt(struct vmxnet3_softc * sc)986 vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc)
987 {
988 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
989 pci_intr_handle_t *intr;
990 void **ihs;
991 struct vmxnet3_queue *vmxq;
992 int i;
993 const char *intrstr;
994 char intrbuf[PCI_INTRSTR_LEN];
995 char xnamebuf[32];
996
997 intr = &sc->vmx_intrs[0];
998 ihs = sc->vmx_ihs;
999 vmxq = &sc->vmx_queue[0];
1000
1001 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
1002
1003 snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev));
1004 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
1005 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
1006 vmxnet3_legacy_intr, sc, xnamebuf);
1007 if (*ihs == NULL) {
1008 aprint_error_dev(sc->vmx_dev,
1009 "unable to establish interrupt at %s\n", intrstr);
1010 return (-1);
1011 }
1012 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr);
1013
1014 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
1015 vmxnet3_handle_queue, vmxq);
1016 if (vmxq->vxq_si == NULL) {
1017 aprint_error_dev(sc->vmx_dev,
1018 "softint_establish for vxq_si failed\n");
1019 return (-1);
1020 }
1021
1022 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++)
1023 sc->vmx_queue[i].vxq_intr_idx = 0;
1024 sc->vmx_event_intr_idx = 0;
1025
1026 return (0);
1027 }
1028
1029 static int
vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc * sc)1030 vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
1031 {
1032 pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
1033 pci_intr_handle_t *intr;
1034 void **ihs;
1035 struct vmxnet3_queue *vmxq;
1036 int i;
1037 const char *intrstr;
1038 char intrbuf[PCI_INTRSTR_LEN];
1039 char xnamebuf[32];
1040
1041 intr = &sc->vmx_intrs[0];
1042 ihs = sc->vmx_ihs;
1043 vmxq = &sc->vmx_queue[0];
1044
1045 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
1046
1047 snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev));
1048 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
1049 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
1050 vmxnet3_legacy_intr, sc, xnamebuf);
1051 if (*ihs == NULL) {
1052 aprint_error_dev(sc->vmx_dev,
1053 "unable to establish interrupt at %s\n", intrstr);
1054 return (-1);
1055 }
1056 aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr);
1057
1058 vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
1059 vmxnet3_handle_queue, vmxq);
1060 if (vmxq->vxq_si == NULL) {
1061 aprint_error_dev(sc->vmx_dev,
1062 "softint_establish for vxq_si failed\n");
1063 return (-1);
1064 }
1065
1066 for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++)
1067 sc->vmx_queue[i].vxq_intr_idx = 0;
1068 sc->vmx_event_intr_idx = 0;
1069
1070 return (0);
1071 }
1072
1073 static void
vmxnet3_set_interrupt_idx(struct vmxnet3_softc * sc)1074 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
1075 {
1076 struct vmxnet3_queue *vmxq;
1077 struct vmxnet3_txqueue *txq;
1078 struct vmxnet3_txq_shared *txs;
1079 struct vmxnet3_rxqueue *rxq;
1080 struct vmxnet3_rxq_shared *rxs;
1081 int i;
1082
1083 sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
1084
1085 for (i = 0; i < sc->vmx_ntxqueues; i++) {
1086 vmxq = &sc->vmx_queue[i];
1087 txq = &vmxq->vxq_txqueue;
1088 txs = txq->vxtxq_ts;
1089 txs->intr_idx = vmxq->vxq_intr_idx;
1090 }
1091
1092 for (i = 0; i < sc->vmx_nrxqueues; i++) {
1093 vmxq = &sc->vmx_queue[i];
1094 rxq = &vmxq->vxq_rxqueue;
1095 rxs = rxq->vxrxq_rs;
1096 rxs->intr_idx = vmxq->vxq_intr_idx;
1097 }
1098 }
1099
1100 static int
vmxnet3_setup_interrupts(struct vmxnet3_softc * sc)1101 vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
1102 {
1103 int error;
1104
1105 switch (sc->vmx_intr_type) {
1106 case VMXNET3_IT_MSIX:
1107 error = vmxnet3_setup_msix_interrupts(sc);
1108 break;
1109 case VMXNET3_IT_MSI:
1110 error = vmxnet3_setup_msi_interrupt(sc);
1111 break;
1112 case VMXNET3_IT_LEGACY:
1113 error = vmxnet3_setup_legacy_interrupt(sc);
1114 break;
1115 default:
1116 panic("%s: invalid interrupt type %d", __func__,
1117 sc->vmx_intr_type);
1118 }
1119
1120 if (error == 0)
1121 vmxnet3_set_interrupt_idx(sc);
1122
1123 return (error);
1124 }
1125
1126 static int
vmxnet3_init_rxq(struct vmxnet3_softc * sc,int q)1127 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
1128 {
1129 struct vmxnet3_rxqueue *rxq;
1130 struct vmxnet3_rxring *rxr;
1131 int i;
1132
1133 rxq = &sc->vmx_queue[q].vxq_rxqueue;
1134
1135 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
1136 device_xname(sc->vmx_dev), q);
1137 rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */);
1138
1139 rxq->vxrxq_sc = sc;
1140
1141 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1142 rxr = &rxq->vxrxq_cmd_ring[i];
1143 rxr->vxrxr_rid = i;
1144 rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
1145 rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc *
1146 sizeof(struct vmxnet3_rxbuf), KM_SLEEP);
1147
1148 rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
1149 }
1150
1151 rxq->vxrxq_stopping = true;
1152
1153 return (0);
1154 }
1155
1156 static int
vmxnet3_init_txq(struct vmxnet3_softc * sc,int q)1157 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
1158 {
1159 struct vmxnet3_txqueue *txq;
1160 struct vmxnet3_txring *txr;
1161
1162 txq = &sc->vmx_queue[q].vxq_txqueue;
1163 txr = &txq->vxtxq_cmd_ring;
1164
1165 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
1166 device_xname(sc->vmx_dev), q);
1167 txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */);
1168
1169 txq->vxtxq_sc = sc;
1170
1171 txq->vxtxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
1172 vmxnet3_deferred_transmit, txq);
1173 if (txq->vxtxq_si == NULL) {
1174 mutex_obj_free(txq->vxtxq_mtx);
1175 aprint_error_dev(sc->vmx_dev,
1176 "softint_establish for vxtxq_si failed\n");
1177 return ENOMEM;
1178 }
1179
1180 txr->vxtxr_ndesc = sc->vmx_ntxdescs;
1181 txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc *
1182 sizeof(struct vmxnet3_txbuf), KM_SLEEP);
1183
1184 txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
1185
1186 txq->vxtxq_interq = pcq_create(sc->vmx_ntxdescs, KM_SLEEP);
1187
1188 txq->vxtxq_stopping = true;
1189
1190 return (0);
1191 }
1192
1193 static int
vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc * sc)1194 vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
1195 {
1196 int i, error, max_nqueues;
1197
1198 KASSERT(!cpu_intr_p());
1199 KASSERT(!cpu_softintr_p());
1200
1201 /*
1202 * Only attempt to create multiple queues if MSIX is available.
1203 * This check prevents us from allocating queue structures that
1204 * we will not use.
1205 *
1206 * FreeBSD:
1207 * MSIX is disabled by default because its apparently broken for
1208 * devices passed through by at least ESXi 5.1.
1209 * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX.
1210 */
1211 if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
1212 sc->vmx_max_nrxqueues = 1;
1213 sc->vmx_max_ntxqueues = 1;
1214 }
1215
1216 max_nqueues = MAX(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues);
1217 sc->vmx_queue = kmem_zalloc(sizeof(struct vmxnet3_queue) * max_nqueues,
1218 KM_SLEEP);
1219
1220 for (i = 0; i < max_nqueues; i++) {
1221 struct vmxnet3_queue *vmxq = &sc->vmx_queue[i];
1222 vmxq->vxq_id = i;
1223 }
1224
1225 for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
1226 error = vmxnet3_init_rxq(sc, i);
1227 if (error)
1228 return (error);
1229 }
1230
1231 for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
1232 error = vmxnet3_init_txq(sc, i);
1233 if (error)
1234 return (error);
1235 }
1236
1237 return (0);
1238 }
1239
1240 static void
vmxnet3_destroy_rxq(struct vmxnet3_rxqueue * rxq)1241 vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
1242 {
1243 struct vmxnet3_rxring *rxr;
1244 int i;
1245
1246 rxq->vxrxq_sc = NULL;
1247
1248 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1249 rxr = &rxq->vxrxq_cmd_ring[i];
1250
1251 if (rxr->vxrxr_rxbuf != NULL) {
1252 kmem_free(rxr->vxrxr_rxbuf,
1253 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf));
1254 rxr->vxrxr_rxbuf = NULL;
1255 }
1256 }
1257
1258 if (rxq->vxrxq_mtx != NULL)
1259 mutex_obj_free(rxq->vxrxq_mtx);
1260 }
1261
1262 static void
vmxnet3_destroy_txq(struct vmxnet3_txqueue * txq)1263 vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
1264 {
1265 struct vmxnet3_txring *txr;
1266 struct mbuf *m;
1267
1268 txr = &txq->vxtxq_cmd_ring;
1269
1270 txq->vxtxq_sc = NULL;
1271
1272 softint_disestablish(txq->vxtxq_si);
1273
1274 while ((m = pcq_get(txq->vxtxq_interq)) != NULL)
1275 m_freem(m);
1276 pcq_destroy(txq->vxtxq_interq);
1277
1278 if (txr->vxtxr_txbuf != NULL) {
1279 kmem_free(txr->vxtxr_txbuf,
1280 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf));
1281 txr->vxtxr_txbuf = NULL;
1282 }
1283
1284 if (txq->vxtxq_mtx != NULL)
1285 mutex_obj_free(txq->vxtxq_mtx);
1286 }
1287
1288 static void
vmxnet3_free_rxtx_queues(struct vmxnet3_softc * sc)1289 vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
1290 {
1291 int i;
1292
1293 if (sc->vmx_queue != NULL) {
1294 int max_nqueues;
1295
1296 for (i = 0; i < sc->vmx_max_nrxqueues; i++)
1297 vmxnet3_destroy_rxq(&sc->vmx_queue[i].vxq_rxqueue);
1298
1299 for (i = 0; i < sc->vmx_max_ntxqueues; i++)
1300 vmxnet3_destroy_txq(&sc->vmx_queue[i].vxq_txqueue);
1301
1302 max_nqueues = MAX(sc->vmx_max_nrxqueues, sc->vmx_max_ntxqueues);
1303 kmem_free(sc->vmx_queue,
1304 sizeof(struct vmxnet3_queue) * max_nqueues);
1305 }
1306 }
1307
1308 static int
vmxnet3_alloc_shared_data(struct vmxnet3_softc * sc)1309 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
1310 {
1311 device_t dev;
1312 uint8_t *kva;
1313 size_t size;
1314 int i, error;
1315
1316 dev = sc->vmx_dev;
1317
1318 size = sizeof(struct vmxnet3_driver_shared);
1319 error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
1320 if (error) {
1321 device_printf(dev, "cannot alloc shared memory\n");
1322 return (error);
1323 }
1324 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
1325
1326 size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
1327 sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
1328 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
1329 if (error) {
1330 device_printf(dev, "cannot alloc queue shared memory\n");
1331 return (error);
1332 }
1333 sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
1334 kva = sc->vmx_qs;
1335
1336 for (i = 0; i < sc->vmx_ntxqueues; i++) {
1337 sc->vmx_queue[i].vxq_txqueue.vxtxq_ts =
1338 (struct vmxnet3_txq_shared *) kva;
1339 kva += sizeof(struct vmxnet3_txq_shared);
1340 }
1341 for (i = 0; i < sc->vmx_nrxqueues; i++) {
1342 sc->vmx_queue[i].vxq_rxqueue.vxrxq_rs =
1343 (struct vmxnet3_rxq_shared *) kva;
1344 kva += sizeof(struct vmxnet3_rxq_shared);
1345 }
1346
1347 if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1348 size = sizeof(struct vmxnet3_rss_shared);
1349 error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
1350 if (error) {
1351 device_printf(dev, "cannot alloc rss shared memory\n");
1352 return (error);
1353 }
1354 sc->vmx_rss =
1355 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
1356 }
1357
1358 return (0);
1359 }
1360
1361 static void
vmxnet3_free_shared_data(struct vmxnet3_softc * sc)1362 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1363 {
1364
1365 if (sc->vmx_rss != NULL) {
1366 vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
1367 sc->vmx_rss = NULL;
1368 }
1369
1370 if (sc->vmx_qs != NULL) {
1371 vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
1372 sc->vmx_qs = NULL;
1373 }
1374
1375 if (sc->vmx_ds != NULL) {
1376 vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
1377 sc->vmx_ds = NULL;
1378 }
1379 }
1380
1381 static int
vmxnet3_alloc_txq_data(struct vmxnet3_softc * sc)1382 vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
1383 {
1384 device_t dev;
1385 struct vmxnet3_txqueue *txq;
1386 struct vmxnet3_txring *txr;
1387 struct vmxnet3_comp_ring *txc;
1388 size_t descsz, compsz;
1389 u_int i;
1390 int q, error;
1391
1392 dev = sc->vmx_dev;
1393
1394 for (q = 0; q < sc->vmx_ntxqueues; q++) {
1395 txq = &sc->vmx_queue[q].vxq_txqueue;
1396 txr = &txq->vxtxq_cmd_ring;
1397 txc = &txq->vxtxq_comp_ring;
1398
1399 descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
1400 compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
1401
1402 error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
1403 if (error) {
1404 device_printf(dev, "cannot alloc Tx descriptors for "
1405 "queue %d error %d\n", q, error);
1406 return (error);
1407 }
1408 txr->vxtxr_txd =
1409 (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
1410
1411 error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
1412 if (error) {
1413 device_printf(dev, "cannot alloc Tx comp descriptors "
1414 "for queue %d error %d\n", q, error);
1415 return (error);
1416 }
1417 txc->vxcr_u.txcd =
1418 (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
1419
1420 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1421 error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE,
1422 VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT,
1423 &txr->vxtxr_txbuf[i].vtxb_dmamap);
1424 if (error) {
1425 device_printf(dev, "unable to create Tx buf "
1426 "dmamap for queue %d idx %d\n", q, i);
1427 return (error);
1428 }
1429 }
1430 }
1431
1432 return (0);
1433 }
1434
1435 static void
vmxnet3_free_txq_data(struct vmxnet3_softc * sc)1436 vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
1437 {
1438 struct vmxnet3_txqueue *txq;
1439 struct vmxnet3_txring *txr;
1440 struct vmxnet3_comp_ring *txc;
1441 struct vmxnet3_txbuf *txb;
1442 u_int i;
1443 int q;
1444
1445 for (q = 0; q < sc->vmx_ntxqueues; q++) {
1446 txq = &sc->vmx_queue[q].vxq_txqueue;
1447 txr = &txq->vxtxq_cmd_ring;
1448 txc = &txq->vxtxq_comp_ring;
1449
1450 for (i = 0; i < txr->vxtxr_ndesc; i++) {
1451 txb = &txr->vxtxr_txbuf[i];
1452 if (txb->vtxb_dmamap != NULL) {
1453 bus_dmamap_destroy(sc->vmx_dmat,
1454 txb->vtxb_dmamap);
1455 txb->vtxb_dmamap = NULL;
1456 }
1457 }
1458
1459 if (txc->vxcr_u.txcd != NULL) {
1460 vmxnet3_dma_free(sc, &txc->vxcr_dma);
1461 txc->vxcr_u.txcd = NULL;
1462 }
1463
1464 if (txr->vxtxr_txd != NULL) {
1465 vmxnet3_dma_free(sc, &txr->vxtxr_dma);
1466 txr->vxtxr_txd = NULL;
1467 }
1468 }
1469 }
1470
1471 static int
vmxnet3_alloc_rxq_data(struct vmxnet3_softc * sc)1472 vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
1473 {
1474 device_t dev;
1475 struct vmxnet3_rxqueue *rxq;
1476 struct vmxnet3_rxring *rxr;
1477 struct vmxnet3_comp_ring *rxc;
1478 int descsz, compsz;
1479 u_int i, j;
1480 int q, error;
1481
1482 dev = sc->vmx_dev;
1483
1484 for (q = 0; q < sc->vmx_nrxqueues; q++) {
1485 rxq = &sc->vmx_queue[q].vxq_rxqueue;
1486 rxc = &rxq->vxrxq_comp_ring;
1487 compsz = 0;
1488
1489 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1490 rxr = &rxq->vxrxq_cmd_ring[i];
1491
1492 descsz = rxr->vxrxr_ndesc *
1493 sizeof(struct vmxnet3_rxdesc);
1494 compsz += rxr->vxrxr_ndesc *
1495 sizeof(struct vmxnet3_rxcompdesc);
1496
1497 error = vmxnet3_dma_malloc(sc, descsz, 512,
1498 &rxr->vxrxr_dma);
1499 if (error) {
1500 device_printf(dev, "cannot allocate Rx "
1501 "descriptors for queue %d/%d error %d\n",
1502 i, q, error);
1503 return (error);
1504 }
1505 rxr->vxrxr_rxd =
1506 (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
1507 }
1508
1509 error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
1510 if (error) {
1511 device_printf(dev, "cannot alloc Rx comp descriptors "
1512 "for queue %d error %d\n", q, error);
1513 return (error);
1514 }
1515 rxc->vxcr_u.rxcd =
1516 (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
1517
1518 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1519 rxr = &rxq->vxrxq_cmd_ring[i];
1520
1521 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1,
1522 JUMBO_LEN, 0, BUS_DMA_NOWAIT,
1523 &rxr->vxrxr_spare_dmap);
1524 if (error) {
1525 device_printf(dev, "unable to create spare "
1526 "dmamap for queue %d/%d error %d\n",
1527 q, i, error);
1528 return (error);
1529 }
1530
1531 for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1532 error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1,
1533 JUMBO_LEN, 0, BUS_DMA_NOWAIT,
1534 &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
1535 if (error) {
1536 device_printf(dev, "unable to create "
1537 "dmamap for queue %d/%d slot %d "
1538 "error %d\n",
1539 q, i, j, error);
1540 return (error);
1541 }
1542 }
1543 }
1544 }
1545
1546 return (0);
1547 }
1548
1549 static void
vmxnet3_free_rxq_data(struct vmxnet3_softc * sc)1550 vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
1551 {
1552 struct vmxnet3_rxqueue *rxq;
1553 struct vmxnet3_rxring *rxr;
1554 struct vmxnet3_comp_ring *rxc;
1555 struct vmxnet3_rxbuf *rxb;
1556 u_int i, j;
1557 int q;
1558
1559 for (q = 0; q < sc->vmx_nrxqueues; q++) {
1560 rxq = &sc->vmx_queue[q].vxq_rxqueue;
1561 rxc = &rxq->vxrxq_comp_ring;
1562
1563 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1564 rxr = &rxq->vxrxq_cmd_ring[i];
1565
1566 if (rxr->vxrxr_spare_dmap != NULL) {
1567 bus_dmamap_destroy(sc->vmx_dmat,
1568 rxr->vxrxr_spare_dmap);
1569 rxr->vxrxr_spare_dmap = NULL;
1570 }
1571
1572 for (j = 0; j < rxr->vxrxr_ndesc; j++) {
1573 rxb = &rxr->vxrxr_rxbuf[j];
1574 if (rxb->vrxb_dmamap != NULL) {
1575 bus_dmamap_destroy(sc->vmx_dmat,
1576 rxb->vrxb_dmamap);
1577 rxb->vrxb_dmamap = NULL;
1578 }
1579 }
1580 }
1581
1582 if (rxc->vxcr_u.rxcd != NULL) {
1583 vmxnet3_dma_free(sc, &rxc->vxcr_dma);
1584 rxc->vxcr_u.rxcd = NULL;
1585 }
1586
1587 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
1588 rxr = &rxq->vxrxq_cmd_ring[i];
1589
1590 if (rxr->vxrxr_rxd != NULL) {
1591 vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
1592 rxr->vxrxr_rxd = NULL;
1593 }
1594 }
1595 }
1596 }
1597
1598 static int
vmxnet3_alloc_queue_data(struct vmxnet3_softc * sc)1599 vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
1600 {
1601 int error;
1602
1603 error = vmxnet3_alloc_txq_data(sc);
1604 if (error)
1605 return (error);
1606
1607 error = vmxnet3_alloc_rxq_data(sc);
1608 if (error)
1609 return (error);
1610
1611 return (0);
1612 }
1613
1614 static void
vmxnet3_free_queue_data(struct vmxnet3_softc * sc)1615 vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
1616 {
1617
1618 if (sc->vmx_queue != NULL) {
1619 vmxnet3_free_rxq_data(sc);
1620 vmxnet3_free_txq_data(sc);
1621 }
1622 }
1623
1624 static int
vmxnet3_alloc_mcast_table(struct vmxnet3_softc * sc)1625 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1626 {
1627 int error;
1628
1629 error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
1630 32, &sc->vmx_mcast_dma);
1631 if (error)
1632 device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1633 else
1634 sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
1635
1636 return (error);
1637 }
1638
1639 static void
vmxnet3_free_mcast_table(struct vmxnet3_softc * sc)1640 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1641 {
1642
1643 if (sc->vmx_mcast != NULL) {
1644 vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
1645 sc->vmx_mcast = NULL;
1646 }
1647 }
1648
1649 static void
vmxnet3_init_shared_data(struct vmxnet3_softc * sc)1650 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1651 {
1652 struct vmxnet3_driver_shared *ds;
1653 struct vmxnet3_txqueue *txq;
1654 struct vmxnet3_txq_shared *txs;
1655 struct vmxnet3_rxqueue *rxq;
1656 struct vmxnet3_rxq_shared *rxs;
1657 int i;
1658
1659 ds = sc->vmx_ds;
1660
1661 /*
1662 * Initialize fields of the shared data that remains the same across
1663 * reinits. Note the shared data is zero'd when allocated.
1664 */
1665
1666 ds->magic = VMXNET3_REV1_MAGIC;
1667
1668 /* DriverInfo */
1669 ds->version = VMXNET3_DRIVER_VERSION;
1670 ds->guest = VMXNET3_GOS_FREEBSD |
1671 #ifdef __LP64__
1672 VMXNET3_GOS_64BIT;
1673 #else
1674 VMXNET3_GOS_32BIT;
1675 #endif
1676 ds->vmxnet3_revision = 1;
1677 ds->upt_version = 1;
1678
1679 /* Misc. conf */
1680 ds->driver_data = vtophys(sc);
1681 ds->driver_data_len = sizeof(struct vmxnet3_softc);
1682 ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
1683 ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
1684 ds->nrxsg_max = sc->vmx_max_rxsegs;
1685
1686 /* RSS conf */
1687 if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1688 ds->rss.version = 1;
1689 ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
1690 ds->rss.len = sc->vmx_rss_dma.dma_size;
1691 }
1692
1693 /* Interrupt control. */
1694 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1695 ds->nintr = sc->vmx_nintrs;
1696 ds->evintr = sc->vmx_event_intr_idx;
1697 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1698
1699 for (i = 0; i < sc->vmx_nintrs; i++)
1700 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1701
1702 /* Receive filter. */
1703 ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
1704 ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
1705
1706 /* Tx queues */
1707 for (i = 0; i < sc->vmx_ntxqueues; i++) {
1708 txq = &sc->vmx_queue[i].vxq_txqueue;
1709 txs = txq->vxtxq_ts;
1710
1711 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
1712 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1713 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
1714 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1715 txs->driver_data = vtophys(txq);
1716 txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1717 }
1718
1719 /* Rx queues */
1720 for (i = 0; i < sc->vmx_nrxqueues; i++) {
1721 rxq = &sc->vmx_queue[i].vxq_rxqueue;
1722 rxs = rxq->vxrxq_rs;
1723
1724 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
1725 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1726 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
1727 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1728 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
1729 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1730 rxs->driver_data = vtophys(rxq);
1731 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1732 }
1733 }
1734
1735 static void
vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc * sc)1736 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1737 {
1738 /*
1739 * Use the same key as the Linux driver until FreeBSD can do
1740 * RSS (presumably Toeplitz) in software.
1741 */
1742 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1743 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1744 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1745 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1746 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1747 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1748 };
1749
1750 struct vmxnet3_rss_shared *rss;
1751 int i;
1752
1753 rss = sc->vmx_rss;
1754
1755 rss->hash_type =
1756 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1757 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1758 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1759 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1760 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1761 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1762
1763 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1764 rss->ind_table[i] = i % sc->vmx_nrxqueues;
1765 }
1766
1767 static void
vmxnet3_reinit_shared_data(struct vmxnet3_softc * sc)1768 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1769 {
1770 struct ifnet *ifp;
1771 struct vmxnet3_driver_shared *ds;
1772
1773 ifp = &sc->vmx_ethercom.ec_if;
1774 ds = sc->vmx_ds;
1775
1776 ds->mtu = ifp->if_mtu;
1777 ds->ntxqueue = sc->vmx_ntxqueues;
1778 ds->nrxqueue = sc->vmx_nrxqueues;
1779
1780 ds->upt_features = 0;
1781 if (ifp->if_capenable &
1782 (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
1783 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx))
1784 ds->upt_features |= UPT1_F_CSUM;
1785 if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING)
1786 ds->upt_features |= UPT1_F_VLAN;
1787
1788 if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1789 ds->upt_features |= UPT1_F_RSS;
1790 vmxnet3_reinit_rss_shared_data(sc);
1791 }
1792
1793 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
1794 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1795 (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
1796 }
1797
1798 static int
vmxnet3_alloc_data(struct vmxnet3_softc * sc)1799 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1800 {
1801 int error;
1802
1803 error = vmxnet3_alloc_shared_data(sc);
1804 if (error)
1805 return (error);
1806
1807 error = vmxnet3_alloc_queue_data(sc);
1808 if (error)
1809 return (error);
1810
1811 error = vmxnet3_alloc_mcast_table(sc);
1812 if (error)
1813 return (error);
1814
1815 vmxnet3_init_shared_data(sc);
1816
1817 return (0);
1818 }
1819
1820 static void
vmxnet3_free_data(struct vmxnet3_softc * sc)1821 vmxnet3_free_data(struct vmxnet3_softc *sc)
1822 {
1823
1824 vmxnet3_free_mcast_table(sc);
1825 vmxnet3_free_queue_data(sc);
1826 vmxnet3_free_shared_data(sc);
1827 }
1828
1829 static int
vmxnet3_setup_interface(struct vmxnet3_softc * sc)1830 vmxnet3_setup_interface(struct vmxnet3_softc *sc)
1831 {
1832 struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
1833
1834 vmxnet3_get_lladdr(sc);
1835 aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n",
1836 ether_sprintf(sc->vmx_lladdr));
1837 vmxnet3_set_lladdr(sc);
1838
1839 strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ);
1840 ifp->if_softc = sc;
1841 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
1842 ifp->if_extflags = IFEF_MPSAFE;
1843 ifp->if_ioctl = vmxnet3_ioctl;
1844 ifp->if_start = vmxnet3_start;
1845 ifp->if_transmit = vmxnet3_transmit;
1846 ifp->if_watchdog = NULL;
1847 ifp->if_init = vmxnet3_init;
1848 ifp->if_stop = vmxnet3_stop;
1849 sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx |
1850 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
1851 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
1852 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
1853 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;
1854
1855 ifp->if_capenable = ifp->if_capabilities;
1856
1857 sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
1858
1859 sc->vmx_ethercom.ec_capabilities |=
1860 ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING | ETHERCAP_JUMBO_MTU;
1861 sc->vmx_ethercom.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;
1862
1863 IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs);
1864 IFQ_SET_READY(&ifp->if_snd);
1865
1866 /* Initialize ifmedia structures. */
1867 sc->vmx_ethercom.ec_ifmedia = &sc->vmx_media;
1868 ifmedia_init_with_lock(&sc->vmx_media, IFM_IMASK, vmxnet3_ifmedia_change,
1869 vmxnet3_ifmedia_status, sc->vmx_mtx);
1870 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
1871 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
1872 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T, 0, NULL);
1873 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
1874 ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T, 0, NULL);
1875 ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);
1876
1877 if_attach(ifp);
1878 if_deferred_start_init(ifp, NULL);
1879 ether_ifattach(ifp, sc->vmx_lladdr);
1880 ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb);
1881 vmxnet3_cmd_link_status(ifp);
1882
1883 /* should set before setting interrupts */
1884 sc->vmx_rx_intr_process_limit = VMXNET3_RX_INTR_PROCESS_LIMIT;
1885 sc->vmx_rx_process_limit = VMXNET3_RX_PROCESS_LIMIT;
1886 sc->vmx_tx_intr_process_limit = VMXNET3_TX_INTR_PROCESS_LIMIT;
1887 sc->vmx_tx_process_limit = VMXNET3_TX_PROCESS_LIMIT;
1888
1889 return (0);
1890 }
1891
1892 static int
vmxnet3_setup_sysctl(struct vmxnet3_softc * sc)1893 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
1894 {
1895 const char *devname;
1896 struct sysctllog **log;
1897 const struct sysctlnode *rnode, *rxnode, *txnode;
1898 int error;
1899
1900 log = &sc->vmx_sysctllog;
1901 devname = device_xname(sc->vmx_dev);
1902
1903 error = sysctl_createv(log, 0, NULL, &rnode,
1904 0, CTLTYPE_NODE, devname,
1905 SYSCTL_DESCR("vmxnet3 information and settings"),
1906 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
1907 if (error)
1908 goto out;
1909 error = sysctl_createv(log, 0, &rnode, NULL,
1910 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
1911 SYSCTL_DESCR("Use workqueue for packet processing"),
1912 NULL, 0, &sc->vmx_txrx_workqueue, 0, CTL_CREATE, CTL_EOL);
1913 if (error)
1914 goto out;
1915
1916 error = sysctl_createv(log, 0, &rnode, &rxnode,
1917 0, CTLTYPE_NODE, "rx",
1918 SYSCTL_DESCR("vmxnet3 information and settings for Rx"),
1919 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
1920 if (error)
1921 goto out;
1922 error = sysctl_createv(log, 0, &rxnode, NULL,
1923 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
1924 SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
1925 NULL, 0, &sc->vmx_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
1926 if (error)
1927 goto out;
1928 error = sysctl_createv(log, 0, &rxnode, NULL,
1929 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
1930 SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
1931 NULL, 0, &sc->vmx_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
1932 if (error)
1933 goto out;
1934
1935 error = sysctl_createv(log, 0, &rnode, &txnode,
1936 0, CTLTYPE_NODE, "tx",
1937 SYSCTL_DESCR("vmxnet3 information and settings for Tx"),
1938 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
1939 if (error)
1940 goto out;
1941 error = sysctl_createv(log, 0, &txnode, NULL,
1942 CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
1943 SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
1944 NULL, 0, &sc->vmx_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
1945 if (error)
1946 goto out;
1947 error = sysctl_createv(log, 0, &txnode, NULL,
1948 CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
1949 SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
1950 NULL, 0, &sc->vmx_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
1951
1952 out:
1953 if (error) {
1954 aprint_error_dev(sc->vmx_dev,
1955 "unable to create sysctl node\n");
1956 sysctl_teardown(log);
1957 }
1958 return error;
1959 }
1960
1961 static int
vmxnet3_setup_stats(struct vmxnet3_softc * sc)1962 vmxnet3_setup_stats(struct vmxnet3_softc *sc)
1963 {
1964 struct vmxnet3_queue *vmxq;
1965 struct vmxnet3_txqueue *txq;
1966 struct vmxnet3_rxqueue *rxq;
1967 int i;
1968
1969 for (i = 0; i < sc->vmx_ntxqueues; i++) {
1970 vmxq = &sc->vmx_queue[i];
1971 txq = &vmxq->vxq_txqueue;
1972 evcnt_attach_dynamic(&txq->vxtxq_intr, EVCNT_TYPE_INTR,
1973 NULL, txq->vxtxq_name, "Interrupt on queue");
1974 evcnt_attach_dynamic(&txq->vxtxq_defer, EVCNT_TYPE_MISC,
1975 NULL, txq->vxtxq_name, "Handled queue in softint/workqueue");
1976 evcnt_attach_dynamic(&txq->vxtxq_deferreq, EVCNT_TYPE_MISC,
1977 NULL, txq->vxtxq_name, "Requested in softint/workqueue");
1978 evcnt_attach_dynamic(&txq->vxtxq_pcqdrop, EVCNT_TYPE_MISC,
1979 NULL, txq->vxtxq_name, "Dropped in pcq");
1980 evcnt_attach_dynamic(&txq->vxtxq_transmitdef, EVCNT_TYPE_MISC,
1981 NULL, txq->vxtxq_name, "Deferred transmit");
1982 evcnt_attach_dynamic(&txq->vxtxq_watchdogto, EVCNT_TYPE_MISC,
1983 NULL, txq->vxtxq_name, "Watchdog timeout");
1984 evcnt_attach_dynamic(&txq->vxtxq_defragged, EVCNT_TYPE_MISC,
1985 NULL, txq->vxtxq_name, "m_defrag successed");
1986 evcnt_attach_dynamic(&txq->vxtxq_defrag_failed, EVCNT_TYPE_MISC,
1987 NULL, txq->vxtxq_name, "m_defrag failed");
1988 }
1989
1990 for (i = 0; i < sc->vmx_nrxqueues; i++) {
1991 vmxq = &sc->vmx_queue[i];
1992 rxq = &vmxq->vxq_rxqueue;
1993 evcnt_attach_dynamic(&rxq->vxrxq_intr, EVCNT_TYPE_INTR,
1994 NULL, rxq->vxrxq_name, "Interrupt on queue");
1995 evcnt_attach_dynamic(&rxq->vxrxq_defer, EVCNT_TYPE_MISC,
1996 NULL, rxq->vxrxq_name, "Handled queue in softint/workqueue");
1997 evcnt_attach_dynamic(&rxq->vxrxq_deferreq, EVCNT_TYPE_MISC,
1998 NULL, rxq->vxrxq_name, "Requested in softint/workqueue");
1999 evcnt_attach_dynamic(&rxq->vxrxq_mgetcl_failed, EVCNT_TYPE_MISC,
2000 NULL, rxq->vxrxq_name, "MCLGET failed");
2001 evcnt_attach_dynamic(&rxq->vxrxq_mbuf_load_failed, EVCNT_TYPE_MISC,
2002 NULL, rxq->vxrxq_name, "bus_dmamap_load_mbuf failed");
2003 }
2004
2005 evcnt_attach_dynamic(&sc->vmx_event_intr, EVCNT_TYPE_INTR,
2006 NULL, device_xname(sc->vmx_dev), "Interrupt for other events");
2007 evcnt_attach_dynamic(&sc->vmx_event_link, EVCNT_TYPE_MISC,
2008 NULL, device_xname(sc->vmx_dev), "Link status event");
2009 evcnt_attach_dynamic(&sc->vmx_event_txqerror, EVCNT_TYPE_MISC,
2010 NULL, device_xname(sc->vmx_dev), "Tx queue error event");
2011 evcnt_attach_dynamic(&sc->vmx_event_rxqerror, EVCNT_TYPE_MISC,
2012 NULL, device_xname(sc->vmx_dev), "Rx queue error event");
2013 evcnt_attach_dynamic(&sc->vmx_event_dic, EVCNT_TYPE_MISC,
2014 NULL, device_xname(sc->vmx_dev), "Device impl change event");
2015 evcnt_attach_dynamic(&sc->vmx_event_debug, EVCNT_TYPE_MISC,
2016 NULL, device_xname(sc->vmx_dev), "Debug event");
2017
2018 return 0;
2019 }
2020
2021 static void
vmxnet3_teardown_stats(struct vmxnet3_softc * sc)2022 vmxnet3_teardown_stats(struct vmxnet3_softc *sc)
2023 {
2024 struct vmxnet3_queue *vmxq;
2025 struct vmxnet3_txqueue *txq;
2026 struct vmxnet3_rxqueue *rxq;
2027 int i;
2028
2029 for (i = 0; i < sc->vmx_ntxqueues; i++) {
2030 vmxq = &sc->vmx_queue[i];
2031 txq = &vmxq->vxq_txqueue;
2032 evcnt_detach(&txq->vxtxq_intr);
2033 evcnt_detach(&txq->vxtxq_defer);
2034 evcnt_detach(&txq->vxtxq_deferreq);
2035 evcnt_detach(&txq->vxtxq_pcqdrop);
2036 evcnt_detach(&txq->vxtxq_transmitdef);
2037 evcnt_detach(&txq->vxtxq_watchdogto);
2038 evcnt_detach(&txq->vxtxq_defragged);
2039 evcnt_detach(&txq->vxtxq_defrag_failed);
2040 }
2041
2042 for (i = 0; i < sc->vmx_nrxqueues; i++) {
2043 vmxq = &sc->vmx_queue[i];
2044 rxq = &vmxq->vxq_rxqueue;
2045 evcnt_detach(&rxq->vxrxq_intr);
2046 evcnt_detach(&rxq->vxrxq_defer);
2047 evcnt_detach(&rxq->vxrxq_deferreq);
2048 evcnt_detach(&rxq->vxrxq_mgetcl_failed);
2049 evcnt_detach(&rxq->vxrxq_mbuf_load_failed);
2050 }
2051
2052 evcnt_detach(&sc->vmx_event_intr);
2053 evcnt_detach(&sc->vmx_event_link);
2054 evcnt_detach(&sc->vmx_event_txqerror);
2055 evcnt_detach(&sc->vmx_event_rxqerror);
2056 evcnt_detach(&sc->vmx_event_dic);
2057 evcnt_detach(&sc->vmx_event_debug);
2058 }
2059
2060 static void
vmxnet3_evintr(struct vmxnet3_softc * sc)2061 vmxnet3_evintr(struct vmxnet3_softc *sc)
2062 {
2063 device_t dev;
2064 struct vmxnet3_txq_shared *ts;
2065 struct vmxnet3_rxq_shared *rs;
2066 uint32_t event;
2067 int reset;
2068
2069 dev = sc->vmx_dev;
2070 reset = 0;
2071
2072 VMXNET3_CORE_LOCK(sc);
2073
2074 /* Clear events. */
2075 event = sc->vmx_ds->event;
2076 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
2077
2078 if (event & VMXNET3_EVENT_LINK) {
2079 sc->vmx_event_link.ev_count++;
2080 vmxnet3_if_link_status(sc);
2081 if (sc->vmx_link_active != 0)
2082 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
2083 }
2084
2085 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
2086 if (event & VMXNET3_EVENT_TQERROR)
2087 sc->vmx_event_txqerror.ev_count++;
2088 if (event & VMXNET3_EVENT_RQERROR)
2089 sc->vmx_event_rxqerror.ev_count++;
2090
2091 reset = 1;
2092 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
2093 ts = sc->vmx_queue[0].vxq_txqueue.vxtxq_ts;
2094 if (ts->stopped != 0)
2095 device_printf(dev, "Tx queue error %#x\n", ts->error);
2096 rs = sc->vmx_queue[0].vxq_rxqueue.vxrxq_rs;
2097 if (rs->stopped != 0)
2098 device_printf(dev, "Rx queue error %#x\n", rs->error);
2099 device_printf(dev, "Rx/Tx queue error event ... resetting\n");
2100 }
2101
2102 if (event & VMXNET3_EVENT_DIC) {
2103 sc->vmx_event_dic.ev_count++;
2104 device_printf(dev, "device implementation change event\n");
2105 }
2106 if (event & VMXNET3_EVENT_DEBUG) {
2107 sc->vmx_event_debug.ev_count++;
2108 device_printf(dev, "debug event\n");
2109 }
2110
2111 if (reset != 0)
2112 vmxnet3_init_locked(sc);
2113
2114 VMXNET3_CORE_UNLOCK(sc);
2115 }
2116
2117 static bool
vmxnet3_txq_eof(struct vmxnet3_txqueue * txq,u_int limit)2118 vmxnet3_txq_eof(struct vmxnet3_txqueue *txq, u_int limit)
2119 {
2120 struct vmxnet3_softc *sc;
2121 struct vmxnet3_txring *txr;
2122 struct vmxnet3_comp_ring *txc;
2123 struct vmxnet3_txcompdesc *txcd;
2124 struct vmxnet3_txbuf *txb;
2125 struct ifnet *ifp;
2126 struct mbuf *m;
2127 u_int sop;
2128 bool more = false;
2129
2130 sc = txq->vxtxq_sc;
2131 txr = &txq->vxtxq_cmd_ring;
2132 txc = &txq->vxtxq_comp_ring;
2133 ifp = &sc->vmx_ethercom.ec_if;
2134
2135 VMXNET3_TXQ_LOCK_ASSERT(txq);
2136
2137 net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
2138 for (;;) {
2139 if (limit-- == 0) {
2140 more = true;
2141 break;
2142 }
2143
2144 txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
2145 if (txcd->gen != txc->vxcr_gen)
2146 break;
2147 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2148
2149 if (++txc->vxcr_next == txc->vxcr_ndesc) {
2150 txc->vxcr_next = 0;
2151 txc->vxcr_gen ^= 1;
2152 }
2153
2154 sop = txr->vxtxr_next;
2155 txb = &txr->vxtxr_txbuf[sop];
2156
2157 if ((m = txb->vtxb_m) != NULL) {
2158 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap,
2159 0, txb->vtxb_dmamap->dm_mapsize,
2160 BUS_DMASYNC_POSTWRITE);
2161 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap);
2162
2163 if_statinc_ref(ifp, nsr, if_opackets);
2164 if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len);
2165 if (m->m_flags & M_MCAST)
2166 if_statinc_ref(ifp, nsr, if_omcasts);
2167
2168 m_freem(m);
2169 txb->vtxb_m = NULL;
2170 }
2171
2172 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
2173 }
2174 IF_STAT_PUTREF(ifp);
2175
2176 if (txr->vxtxr_head == txr->vxtxr_next)
2177 txq->vxtxq_watchdog = 0;
2178
2179 return more;
2180 }
2181
2182 static int
vmxnet3_newbuf(struct vmxnet3_softc * sc,struct vmxnet3_rxqueue * rxq,struct vmxnet3_rxring * rxr)2183 vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq,
2184 struct vmxnet3_rxring *rxr)
2185 {
2186 struct mbuf *m;
2187 struct vmxnet3_rxdesc *rxd;
2188 struct vmxnet3_rxbuf *rxb;
2189 bus_dma_tag_t tag;
2190 bus_dmamap_t dmap;
2191 int idx, btype, error;
2192
2193 tag = sc->vmx_dmat;
2194 dmap = rxr->vxrxr_spare_dmap;
2195 idx = rxr->vxrxr_fill;
2196 rxd = &rxr->vxrxr_rxd[idx];
2197 rxb = &rxr->vxrxr_rxbuf[idx];
2198
2199 /* Don't allocate buffers for ring 2 for now. */
2200 if (rxr->vxrxr_rid != 0)
2201 return -1;
2202 btype = VMXNET3_BTYPE_HEAD;
2203
2204 MGETHDR(m, M_DONTWAIT, MT_DATA);
2205 if (m == NULL)
2206 return (ENOBUFS);
2207
2208 MCLGET(m, M_DONTWAIT);
2209 if ((m->m_flags & M_EXT) == 0) {
2210 rxq->vxrxq_mgetcl_failed.ev_count++;
2211 m_freem(m);
2212 return (ENOBUFS);
2213 }
2214
2215 m->m_pkthdr.len = m->m_len = JUMBO_LEN;
2216 m_adj(m, ETHER_ALIGN);
2217
2218 error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT);
2219 if (error) {
2220 m_freem(m);
2221 rxq->vxrxq_mbuf_load_failed.ev_count++;
2222 return (error);
2223 }
2224
2225 if (rxb->vrxb_m != NULL) {
2226 bus_dmamap_sync(tag, rxb->vrxb_dmamap,
2227 0, rxb->vrxb_dmamap->dm_mapsize,
2228 BUS_DMASYNC_POSTREAD);
2229 bus_dmamap_unload(tag, rxb->vrxb_dmamap);
2230 }
2231
2232 rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
2233 rxb->vrxb_dmamap = dmap;
2234 rxb->vrxb_m = m;
2235
2236 rxd->addr = DMAADDR(dmap);
2237 rxd->len = m->m_pkthdr.len;
2238 rxd->btype = btype;
2239 rxd->gen = rxr->vxrxr_gen;
2240
2241 vmxnet3_rxr_increment_fill(rxr);
2242 return (0);
2243 }
2244
2245 static void
vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue * rxq,struct vmxnet3_rxring * rxr,int idx)2246 vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
2247 struct vmxnet3_rxring *rxr, int idx)
2248 {
2249 struct vmxnet3_rxdesc *rxd;
2250
2251 rxd = &rxr->vxrxr_rxd[idx];
2252 rxd->gen = rxr->vxrxr_gen;
2253 vmxnet3_rxr_increment_fill(rxr);
2254 }
2255
2256 static void
vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue * rxq)2257 vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
2258 {
2259 struct vmxnet3_softc *sc;
2260 struct vmxnet3_rxring *rxr;
2261 struct vmxnet3_comp_ring *rxc;
2262 struct vmxnet3_rxcompdesc *rxcd;
2263 int idx, eof;
2264
2265 sc = rxq->vxrxq_sc;
2266 rxc = &rxq->vxrxq_comp_ring;
2267
2268 do {
2269 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2270 if (rxcd->gen != rxc->vxcr_gen)
2271 break; /* Not expected. */
2272 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2273
2274 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2275 rxc->vxcr_next = 0;
2276 rxc->vxcr_gen ^= 1;
2277 }
2278
2279 idx = rxcd->rxd_idx;
2280 eof = rxcd->eop;
2281 if (rxcd->qid < sc->vmx_nrxqueues)
2282 rxr = &rxq->vxrxq_cmd_ring[0];
2283 else
2284 rxr = &rxq->vxrxq_cmd_ring[1];
2285 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2286 } while (!eof);
2287 }
2288
2289 static void
vmxnet3_rx_csum(struct vmxnet3_rxcompdesc * rxcd,struct mbuf * m)2290 vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2291 {
2292 if (rxcd->no_csum)
2293 return;
2294
2295 if (rxcd->ipv4) {
2296 m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
2297 if (rxcd->ipcsum_ok == 0)
2298 m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD;
2299 }
2300
2301 if (rxcd->fragment)
2302 return;
2303
2304 if (rxcd->tcp) {
2305 m->m_pkthdr.csum_flags |=
2306 rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6;
2307 if ((rxcd->csum_ok) == 0)
2308 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
2309 }
2310
2311 if (rxcd->udp) {
2312 m->m_pkthdr.csum_flags |=
2313 rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ;
2314 if ((rxcd->csum_ok) == 0)
2315 m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
2316 }
2317 }
2318
2319 static void
vmxnet3_rxq_input(struct vmxnet3_rxqueue * rxq,struct vmxnet3_rxcompdesc * rxcd,struct mbuf * m)2320 vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
2321 struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
2322 {
2323 struct vmxnet3_softc *sc;
2324 struct ifnet *ifp;
2325
2326 sc = rxq->vxrxq_sc;
2327 ifp = &sc->vmx_ethercom.ec_if;
2328
2329 if (rxcd->error) {
2330 if_statinc(ifp, if_ierrors);
2331 m_freem(m);
2332 return;
2333 }
2334
2335 if (!rxcd->no_csum)
2336 vmxnet3_rx_csum(rxcd, m);
2337 if (rxcd->vlan)
2338 vlan_set_tag(m, rxcd->vtag);
2339
2340 net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
2341 if_statinc_ref(ifp, nsr, if_ipackets);
2342 if_statadd_ref(ifp, nsr, if_ibytes, m->m_pkthdr.len);
2343 IF_STAT_PUTREF(ifp);
2344
2345 if_percpuq_enqueue(ifp->if_percpuq, m);
2346 }
2347
2348 static bool
vmxnet3_rxq_eof(struct vmxnet3_rxqueue * rxq,u_int limit)2349 vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq, u_int limit)
2350 {
2351 struct vmxnet3_softc *sc;
2352 struct ifnet *ifp;
2353 struct vmxnet3_rxring *rxr;
2354 struct vmxnet3_comp_ring *rxc;
2355 struct vmxnet3_rxdesc *rxd __diagused;
2356 struct vmxnet3_rxcompdesc *rxcd;
2357 struct mbuf *m, *m_head, *m_tail;
2358 u_int idx, length;
2359 bool more = false;
2360
2361 sc = rxq->vxrxq_sc;
2362 ifp = &sc->vmx_ethercom.ec_if;
2363 rxc = &rxq->vxrxq_comp_ring;
2364
2365 VMXNET3_RXQ_LOCK_ASSERT(rxq);
2366
2367 if (rxq->vxrxq_stopping)
2368 return more;
2369
2370 m_head = rxq->vxrxq_mhead;
2371 rxq->vxrxq_mhead = NULL;
2372 m_tail = rxq->vxrxq_mtail;
2373 rxq->vxrxq_mtail = NULL;
2374 KASSERT(m_head == NULL || m_tail != NULL);
2375
2376 for (;;) {
2377 if (limit-- == 0) {
2378 more = true;
2379 break;
2380 }
2381
2382 rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
2383 if (rxcd->gen != rxc->vxcr_gen) {
2384 rxq->vxrxq_mhead = m_head;
2385 rxq->vxrxq_mtail = m_tail;
2386 break;
2387 }
2388 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
2389
2390 if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
2391 rxc->vxcr_next = 0;
2392 rxc->vxcr_gen ^= 1;
2393 }
2394
2395 idx = rxcd->rxd_idx;
2396 length = rxcd->len;
2397 if (rxcd->qid < sc->vmx_nrxqueues)
2398 rxr = &rxq->vxrxq_cmd_ring[0];
2399 else
2400 rxr = &rxq->vxrxq_cmd_ring[1];
2401 rxd = &rxr->vxrxr_rxd[idx];
2402
2403 m = rxr->vxrxr_rxbuf[idx].vrxb_m;
2404 KASSERT(m != NULL);
2405
2406 /*
2407 * The host may skip descriptors. We detect this when this
2408 * descriptor does not match the previous fill index. Catch
2409 * up with the host now.
2410 */
2411 if (__predict_false(rxr->vxrxr_fill != idx)) {
2412 while (rxr->vxrxr_fill != idx) {
2413 rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
2414 rxr->vxrxr_gen;
2415 vmxnet3_rxr_increment_fill(rxr);
2416 }
2417 }
2418
2419 if (rxcd->sop) {
2420 /* start of frame w/o head buffer */
2421 KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD);
2422 /* start of frame not in ring 0 */
2423 KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]);
2424 /* duplicate start of frame? */
2425 KASSERT(m_head == NULL);
2426
2427 if (length == 0) {
2428 /* Just ignore this descriptor. */
2429 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2430 goto nextp;
2431 }
2432
2433 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) {
2434 if_statinc(ifp, if_iqdrops);
2435 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2436 if (!rxcd->eop)
2437 vmxnet3_rxq_discard_chain(rxq);
2438 goto nextp;
2439 }
2440
2441 m_set_rcvif(m, ifp);
2442 m->m_pkthdr.len = m->m_len = length;
2443 m->m_pkthdr.csum_flags = 0;
2444 m_head = m_tail = m;
2445
2446 } else {
2447 /* non start of frame w/o body buffer */
2448 KASSERT(rxd->btype == VMXNET3_BTYPE_BODY);
2449 /* frame not started? */
2450 KASSERT(m_head != NULL);
2451
2452 if (vmxnet3_newbuf(sc, rxq, rxr) != 0) {
2453 if_statinc(ifp, if_iqdrops);
2454 vmxnet3_rxq_eof_discard(rxq, rxr, idx);
2455 if (!rxcd->eop)
2456 vmxnet3_rxq_discard_chain(rxq);
2457 m_freem(m_head);
2458 m_head = m_tail = NULL;
2459 goto nextp;
2460 }
2461
2462 m->m_len = length;
2463 m_head->m_pkthdr.len += length;
2464 m_tail->m_next = m;
2465 m_tail = m;
2466 }
2467
2468 if (rxcd->eop) {
2469 vmxnet3_rxq_input(rxq, rxcd, m_head);
2470 m_head = m_tail = NULL;
2471
2472 /* Must recheck after dropping the Rx lock. */
2473 if (rxq->vxrxq_stopping)
2474 break;
2475 }
2476
2477 nextp:
2478 if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
2479 int qid = rxcd->qid;
2480 bus_size_t r;
2481
2482 idx = (idx + 1) % rxr->vxrxr_ndesc;
2483 if (qid >= sc->vmx_nrxqueues) {
2484 qid -= sc->vmx_nrxqueues;
2485 r = VMXNET3_BAR0_RXH2(qid);
2486 } else
2487 r = VMXNET3_BAR0_RXH1(qid);
2488 vmxnet3_write_bar0(sc, r, idx);
2489 }
2490 }
2491
2492 return more;
2493 }
2494
2495 static inline void
vmxnet3_sched_handle_queue(struct vmxnet3_softc * sc,struct vmxnet3_queue * vmxq)2496 vmxnet3_sched_handle_queue(struct vmxnet3_softc *sc, struct vmxnet3_queue *vmxq)
2497 {
2498
2499 if (vmxq->vxq_workqueue) {
2500 /*
2501 * When this function is called, "vmxq" is owned by one CPU.
2502 * so, atomic operation is not required here.
2503 */
2504 if (!vmxq->vxq_wq_enqueued) {
2505 vmxq->vxq_wq_enqueued = true;
2506 workqueue_enqueue(sc->vmx_queue_wq,
2507 &vmxq->vxq_wq_cookie, curcpu());
2508 }
2509 } else {
2510 softint_schedule(vmxq->vxq_si);
2511 }
2512 }
2513
2514 static int
vmxnet3_legacy_intr(void * xsc)2515 vmxnet3_legacy_intr(void *xsc)
2516 {
2517 struct vmxnet3_softc *sc;
2518 struct vmxnet3_queue *vmxq;
2519 struct vmxnet3_txqueue *txq;
2520 struct vmxnet3_rxqueue *rxq;
2521 u_int txlimit, rxlimit;
2522 bool txmore, rxmore;
2523
2524 sc = xsc;
2525 vmxq = &sc->vmx_queue[0];
2526 txq = &vmxq->vxq_txqueue;
2527 rxq = &vmxq->vxq_rxqueue;
2528 txlimit = sc->vmx_tx_intr_process_limit;
2529 rxlimit = sc->vmx_rx_intr_process_limit;
2530
2531 if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
2532 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
2533 return (0);
2534 }
2535 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2536 vmxnet3_disable_all_intrs(sc);
2537
2538 if (sc->vmx_ds->event != 0)
2539 vmxnet3_evintr(sc);
2540
2541 VMXNET3_TXQ_LOCK(txq);
2542 txmore = vmxnet3_txq_eof(txq, txlimit);
2543 VMXNET3_TXQ_UNLOCK(txq);
2544
2545 VMXNET3_RXQ_LOCK(rxq);
2546 rxmore = vmxnet3_rxq_eof(rxq, rxlimit);
2547 VMXNET3_RXQ_UNLOCK(rxq);
2548
2549 if (txmore || rxmore)
2550 vmxnet3_sched_handle_queue(sc, vmxq);
2551 else {
2552 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
2553 vmxnet3_enable_all_intrs(sc);
2554 }
2555
2556 return (1);
2557 }
2558
2559 static int
vmxnet3_txrxq_intr(void * xvmxq)2560 vmxnet3_txrxq_intr(void *xvmxq)
2561 {
2562 struct vmxnet3_softc *sc;
2563 struct vmxnet3_queue *vmxq;
2564 struct vmxnet3_txqueue *txq;
2565 struct vmxnet3_rxqueue *rxq;
2566 u_int txlimit, rxlimit;
2567 bool txmore, rxmore;
2568
2569 vmxq = xvmxq;
2570 txq = &vmxq->vxq_txqueue;
2571 rxq = &vmxq->vxq_rxqueue;
2572 sc = txq->vxtxq_sc;
2573 txlimit = sc->vmx_tx_intr_process_limit;
2574 rxlimit = sc->vmx_rx_intr_process_limit;
2575 vmxq->vxq_workqueue = sc->vmx_txrx_workqueue;
2576
2577 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2578 vmxnet3_disable_intr(sc, vmxq->vxq_intr_idx);
2579
2580 VMXNET3_TXQ_LOCK(txq);
2581 txq->vxtxq_intr.ev_count++;
2582 txmore = vmxnet3_txq_eof(txq, txlimit);
2583 VMXNET3_TXQ_UNLOCK(txq);
2584
2585 VMXNET3_RXQ_LOCK(rxq);
2586 rxq->vxrxq_intr.ev_count++;
2587 rxmore = vmxnet3_rxq_eof(rxq, rxlimit);
2588 VMXNET3_RXQ_UNLOCK(rxq);
2589
2590 if (txmore || rxmore)
2591 vmxnet3_sched_handle_queue(sc, vmxq);
2592 else {
2593 /* for ALTQ */
2594 if (vmxq->vxq_id == 0)
2595 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
2596 softint_schedule(txq->vxtxq_si);
2597
2598 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx);
2599 }
2600
2601 return (1);
2602 }
2603
2604 static void
vmxnet3_handle_queue(void * xvmxq)2605 vmxnet3_handle_queue(void *xvmxq)
2606 {
2607 struct vmxnet3_softc *sc;
2608 struct vmxnet3_queue *vmxq;
2609 struct vmxnet3_txqueue *txq;
2610 struct vmxnet3_rxqueue *rxq;
2611 u_int txlimit, rxlimit;
2612 bool txmore, rxmore;
2613
2614 vmxq = xvmxq;
2615 txq = &vmxq->vxq_txqueue;
2616 rxq = &vmxq->vxq_rxqueue;
2617 sc = txq->vxtxq_sc;
2618 txlimit = sc->vmx_tx_process_limit;
2619 rxlimit = sc->vmx_rx_process_limit;
2620
2621 VMXNET3_TXQ_LOCK(txq);
2622 txq->vxtxq_defer.ev_count++;
2623 txmore = vmxnet3_txq_eof(txq, txlimit);
2624 if (txmore)
2625 txq->vxtxq_deferreq.ev_count++;
2626 /* for ALTQ */
2627 if (vmxq->vxq_id == 0)
2628 if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
2629 softint_schedule(txq->vxtxq_si);
2630 VMXNET3_TXQ_UNLOCK(txq);
2631
2632 VMXNET3_RXQ_LOCK(rxq);
2633 rxq->vxrxq_defer.ev_count++;
2634 rxmore = vmxnet3_rxq_eof(rxq, rxlimit);
2635 if (rxmore)
2636 rxq->vxrxq_deferreq.ev_count++;
2637 VMXNET3_RXQ_UNLOCK(rxq);
2638
2639 if (txmore || rxmore)
2640 vmxnet3_sched_handle_queue(sc, vmxq);
2641 else
2642 vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx);
2643 }
2644
2645 static void
vmxnet3_handle_queue_work(struct work * wk,void * context)2646 vmxnet3_handle_queue_work(struct work *wk, void *context)
2647 {
2648 struct vmxnet3_queue *vmxq;
2649
2650 vmxq = container_of(wk, struct vmxnet3_queue, vxq_wq_cookie);
2651 vmxq->vxq_wq_enqueued = false;
2652 vmxnet3_handle_queue(vmxq);
2653 }
2654
2655 static int
vmxnet3_event_intr(void * xsc)2656 vmxnet3_event_intr(void *xsc)
2657 {
2658 struct vmxnet3_softc *sc;
2659
2660 sc = xsc;
2661
2662 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
2663 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
2664
2665 sc->vmx_event_intr.ev_count++;
2666
2667 if (sc->vmx_ds->event != 0)
2668 vmxnet3_evintr(sc);
2669
2670 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2671
2672 return (1);
2673 }
2674
2675 static void
vmxnet3_txstop(struct vmxnet3_softc * sc,struct vmxnet3_txqueue * txq)2676 vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2677 {
2678 struct vmxnet3_txring *txr;
2679 struct vmxnet3_txbuf *txb;
2680 u_int i;
2681
2682 txr = &txq->vxtxq_cmd_ring;
2683
2684 for (i = 0; i < txr->vxtxr_ndesc; i++) {
2685 txb = &txr->vxtxr_txbuf[i];
2686
2687 if (txb->vtxb_m == NULL)
2688 continue;
2689
2690 bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap,
2691 0, txb->vtxb_dmamap->dm_mapsize,
2692 BUS_DMASYNC_POSTWRITE);
2693 bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap);
2694 m_freem(txb->vtxb_m);
2695 txb->vtxb_m = NULL;
2696 }
2697 }
2698
2699 static void
vmxnet3_rxstop(struct vmxnet3_softc * sc,struct vmxnet3_rxqueue * rxq)2700 vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2701 {
2702 struct vmxnet3_rxring *rxr;
2703 struct vmxnet3_rxbuf *rxb;
2704 u_int i, j;
2705
2706 if (rxq->vxrxq_mhead != NULL) {
2707 m_freem(rxq->vxrxq_mhead);
2708 rxq->vxrxq_mhead = NULL;
2709 rxq->vxrxq_mtail = NULL;
2710 }
2711
2712 for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
2713 rxr = &rxq->vxrxq_cmd_ring[i];
2714
2715 for (j = 0; j < rxr->vxrxr_ndesc; j++) {
2716 rxb = &rxr->vxrxr_rxbuf[j];
2717
2718 if (rxb->vrxb_m == NULL)
2719 continue;
2720
2721 bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap,
2722 0, rxb->vrxb_dmamap->dm_mapsize,
2723 BUS_DMASYNC_POSTREAD);
2724 bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap);
2725 m_freem(rxb->vrxb_m);
2726 rxb->vrxb_m = NULL;
2727 }
2728 }
2729 }
2730
2731 static void
vmxnet3_stop_rendezvous(struct vmxnet3_softc * sc)2732 vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
2733 {
2734 struct vmxnet3_rxqueue *rxq;
2735 struct vmxnet3_txqueue *txq;
2736 struct vmxnet3_queue *vmxq;
2737 int i;
2738
2739 for (i = 0; i < sc->vmx_nrxqueues; i++) {
2740 rxq = &sc->vmx_queue[i].vxq_rxqueue;
2741 VMXNET3_RXQ_LOCK(rxq);
2742 rxq->vxrxq_stopping = true;
2743 VMXNET3_RXQ_UNLOCK(rxq);
2744 }
2745 for (i = 0; i < sc->vmx_ntxqueues; i++) {
2746 txq = &sc->vmx_queue[i].vxq_txqueue;
2747 VMXNET3_TXQ_LOCK(txq);
2748 txq->vxtxq_stopping = true;
2749 VMXNET3_TXQ_UNLOCK(txq);
2750 }
2751 for (i = 0; i < sc->vmx_nrxqueues; i++) {
2752 vmxq = &sc->vmx_queue[i];
2753 workqueue_wait(sc->vmx_queue_wq, &vmxq->vxq_wq_cookie);
2754 }
2755 }
2756
2757 static void
vmxnet3_stop_locked(struct vmxnet3_softc * sc)2758 vmxnet3_stop_locked(struct vmxnet3_softc *sc)
2759 {
2760 struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
2761 int q;
2762
2763 VMXNET3_CORE_LOCK_ASSERT(sc);
2764 KASSERT(IFNET_LOCKED(ifp));
2765
2766 vmxnet3_stop_rendezvous(sc);
2767
2768 sc->vmx_mcastactive = false;
2769 sc->vmx_link_active = 0;
2770 callout_halt(&sc->vmx_tick, sc->vmx_mtx);
2771
2772 ifp->if_flags &= ~IFF_RUNNING;
2773
2774 /* Disable interrupts. */
2775 vmxnet3_disable_all_intrs(sc);
2776 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
2777
2778 for (q = 0; q < sc->vmx_ntxqueues; q++)
2779 vmxnet3_txstop(sc, &sc->vmx_queue[q].vxq_txqueue);
2780 for (q = 0; q < sc->vmx_nrxqueues; q++)
2781 vmxnet3_rxstop(sc, &sc->vmx_queue[q].vxq_rxqueue);
2782
2783 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
2784 }
2785
2786 static void
vmxnet3_stop(struct ifnet * ifp,int disable)2787 vmxnet3_stop(struct ifnet *ifp, int disable)
2788 {
2789 struct vmxnet3_softc *sc = ifp->if_softc;
2790
2791 KASSERT(IFNET_LOCKED(ifp));
2792
2793 VMXNET3_CORE_LOCK(sc);
2794 vmxnet3_stop_locked(sc);
2795 VMXNET3_CORE_UNLOCK(sc);
2796 }
2797
2798 static void
vmxnet3_txinit(struct vmxnet3_softc * sc,struct vmxnet3_txqueue * txq)2799 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
2800 {
2801 struct vmxnet3_txring *txr;
2802 struct vmxnet3_comp_ring *txc;
2803
2804 txr = &txq->vxtxq_cmd_ring;
2805 txr->vxtxr_head = 0;
2806 txr->vxtxr_next = 0;
2807 txr->vxtxr_gen = VMXNET3_INIT_GEN;
2808 memset(txr->vxtxr_txd, 0,
2809 txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
2810
2811 txc = &txq->vxtxq_comp_ring;
2812 txc->vxcr_next = 0;
2813 txc->vxcr_gen = VMXNET3_INIT_GEN;
2814 memset(txc->vxcr_u.txcd, 0,
2815 txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
2816 }
2817
2818 static int
vmxnet3_rxinit(struct vmxnet3_softc * sc,struct vmxnet3_rxqueue * rxq)2819 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
2820 {
2821 struct vmxnet3_rxring *rxr;
2822 struct vmxnet3_comp_ring *rxc;
2823 u_int i, populate, idx;
2824 int error;
2825
2826 /* LRO and jumbo frame is not supported yet */
2827 populate = 1;
2828
2829 for (i = 0; i < populate; i++) {
2830 rxr = &rxq->vxrxq_cmd_ring[i];
2831 rxr->vxrxr_fill = 0;
2832 rxr->vxrxr_gen = VMXNET3_INIT_GEN;
2833 memset(rxr->vxrxr_rxd, 0,
2834 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2835
2836 for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
2837 error = vmxnet3_newbuf(sc, rxq, rxr);
2838 if (error)
2839 return (error);
2840 }
2841 }
2842
2843 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
2844 rxr = &rxq->vxrxq_cmd_ring[i];
2845 rxr->vxrxr_fill = 0;
2846 rxr->vxrxr_gen = 0;
2847 memset(rxr->vxrxr_rxd, 0,
2848 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
2849 }
2850
2851 rxc = &rxq->vxrxq_comp_ring;
2852 rxc->vxcr_next = 0;
2853 rxc->vxcr_gen = VMXNET3_INIT_GEN;
2854 memset(rxc->vxcr_u.rxcd, 0,
2855 rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
2856
2857 return (0);
2858 }
2859
2860 static int
vmxnet3_reinit_queues(struct vmxnet3_softc * sc)2861 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
2862 {
2863 device_t dev;
2864 int q, error;
2865 dev = sc->vmx_dev;
2866
2867 for (q = 0; q < sc->vmx_ntxqueues; q++)
2868 vmxnet3_txinit(sc, &sc->vmx_queue[q].vxq_txqueue);
2869
2870 for (q = 0; q < sc->vmx_nrxqueues; q++) {
2871 error = vmxnet3_rxinit(sc, &sc->vmx_queue[q].vxq_rxqueue);
2872 if (error) {
2873 device_printf(dev, "cannot populate Rx queue %d\n", q);
2874 return (error);
2875 }
2876 }
2877
2878 return (0);
2879 }
2880
2881 static int
vmxnet3_enable_device(struct vmxnet3_softc * sc)2882 vmxnet3_enable_device(struct vmxnet3_softc *sc)
2883 {
2884 int q;
2885
2886 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
2887 device_printf(sc->vmx_dev, "device enable command failed!\n");
2888 return (1);
2889 }
2890
2891 /* Reset the Rx queue heads. */
2892 for (q = 0; q < sc->vmx_nrxqueues; q++) {
2893 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
2894 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
2895 }
2896
2897 return (0);
2898 }
2899
2900 static void
vmxnet3_reinit_rxfilters(struct vmxnet3_softc * sc)2901 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
2902 {
2903
2904 vmxnet3_set_rxfilter(sc);
2905
2906 memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter));
2907 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
2908 }
2909
2910 static int
vmxnet3_reinit(struct vmxnet3_softc * sc)2911 vmxnet3_reinit(struct vmxnet3_softc *sc)
2912 {
2913
2914 VMXNET3_CORE_LOCK_ASSERT(sc);
2915
2916 vmxnet3_set_lladdr(sc);
2917 vmxnet3_reinit_shared_data(sc);
2918
2919 if (vmxnet3_reinit_queues(sc) != 0)
2920 return (ENXIO);
2921
2922 if (vmxnet3_enable_device(sc) != 0)
2923 return (ENXIO);
2924
2925 vmxnet3_reinit_rxfilters(sc);
2926
2927 return (0);
2928 }
2929
2930 static int
vmxnet3_init_locked(struct vmxnet3_softc * sc)2931 vmxnet3_init_locked(struct vmxnet3_softc *sc)
2932 {
2933 struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
2934 int q;
2935 int error;
2936
2937 KASSERT(IFNET_LOCKED(ifp));
2938 VMXNET3_CORE_LOCK_ASSERT(sc);
2939
2940 vmxnet3_stop_locked(sc);
2941
2942 error = vmxnet3_reinit(sc);
2943 if (error) {
2944 vmxnet3_stop_locked(sc);
2945 return (error);
2946 }
2947
2948 ifp->if_flags |= IFF_RUNNING;
2949 vmxnet3_if_link_status(sc);
2950 sc->vmx_mcastactive = true;
2951
2952 vmxnet3_enable_all_intrs(sc);
2953 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
2954
2955 for (q = 0; q < sc->vmx_ntxqueues; q++) {
2956 VMXNET3_TXQ_LOCK(&sc->vmx_queue[q].vxq_txqueue);
2957 sc->vmx_queue[q].vxq_txqueue.vxtxq_stopping = false;
2958 VMXNET3_TXQ_UNLOCK(&sc->vmx_queue[q].vxq_txqueue);
2959 }
2960 for (q = 0; q < sc->vmx_nrxqueues; q++) {
2961 VMXNET3_RXQ_LOCK(&sc->vmx_queue[q].vxq_rxqueue);
2962 sc->vmx_queue[q].vxq_rxqueue.vxrxq_stopping = false;
2963 VMXNET3_RXQ_UNLOCK(&sc->vmx_queue[q].vxq_rxqueue);
2964 }
2965
2966 return (0);
2967 }
2968
2969 static int
vmxnet3_init(struct ifnet * ifp)2970 vmxnet3_init(struct ifnet *ifp)
2971 {
2972 struct vmxnet3_softc *sc = ifp->if_softc;
2973 int error;
2974
2975 KASSERT(IFNET_LOCKED(ifp));
2976
2977 VMXNET3_CORE_LOCK(sc);
2978 error = vmxnet3_init_locked(sc);
2979 VMXNET3_CORE_UNLOCK(sc);
2980
2981 return (error);
2982 }
2983
2984 static int
vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue * txq,struct mbuf * m,int * start,int * csum_start)2985 vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
2986 int *start, int *csum_start)
2987 {
2988 struct ether_header *eh;
2989 struct mbuf *mp;
2990 int offset, csum_off, iphl, offp;
2991 bool v4;
2992
2993 eh = mtod(m, struct ether_header *);
2994 switch (htons(eh->ether_type)) {
2995 case ETHERTYPE_IP:
2996 case ETHERTYPE_IPV6:
2997 offset = ETHER_HDR_LEN;
2998 break;
2999 case ETHERTYPE_VLAN:
3000 offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3001 break;
3002 default:
3003 m_freem(m);
3004 return (EINVAL);
3005 }
3006
3007 if ((m->m_pkthdr.csum_flags &
3008 (M_CSUM_TSOv4 | M_CSUM_UDPv4 | M_CSUM_TCPv4)) != 0) {
3009 iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data);
3010 v4 = true;
3011 } else {
3012 iphl = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data);
3013 v4 = false;
3014 }
3015 *start = offset + iphl;
3016
3017 if (m->m_pkthdr.csum_flags &
3018 (M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
3019 csum_off = offsetof(struct tcphdr, th_sum);
3020 } else {
3021 csum_off = offsetof(struct udphdr, uh_sum);
3022 }
3023
3024 *csum_start = *start + csum_off;
3025 mp = m_pulldown(m, 0, *csum_start + 2, &offp);
3026 if (!mp) {
3027 /* m is already freed */
3028 return ENOBUFS;
3029 }
3030
3031 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
3032 struct tcphdr *tcp;
3033
3034 txq->vxtxq_stats.vmtxs_tso++;
3035 tcp = (void *)(mtod(mp, char *) + offp + *start);
3036
3037 if (v4) {
3038 struct ip *ip;
3039
3040 ip = (void *)(mtod(mp, char *) + offp + offset);
3041 tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr,
3042 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3043 } else {
3044 struct ip6_hdr *ip6;
3045
3046 ip6 = (void *)(mtod(mp, char *) + offp + offset);
3047 tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src,
3048 &ip6->ip6_dst, 0, htonl(IPPROTO_TCP));
3049 }
3050
3051 /*
3052 * For TSO, the size of the protocol header is also
3053 * included in the descriptor header size.
3054 */
3055 *start += (tcp->th_off << 2);
3056 } else
3057 txq->vxtxq_stats.vmtxs_csum++;
3058
3059 return (0);
3060 }
3061
3062 static int
vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue * txq,struct mbuf ** m0,bus_dmamap_t dmap)3063 vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
3064 bus_dmamap_t dmap)
3065 {
3066 struct mbuf *m;
3067 bus_dma_tag_t tag;
3068 int error;
3069
3070 m = *m0;
3071 tag = txq->vxtxq_sc->vmx_dmat;
3072
3073 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT);
3074 if (error == 0 || error != EFBIG)
3075 return (error);
3076
3077 m = m_defrag(m, M_NOWAIT);
3078 if (m != NULL) {
3079 *m0 = m;
3080 error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT);
3081 } else
3082 error = ENOBUFS;
3083
3084 if (error) {
3085 m_freem(*m0);
3086 *m0 = NULL;
3087 txq->vxtxq_defrag_failed.ev_count++;
3088 } else
3089 txq->vxtxq_defragged.ev_count++;
3090
3091 return (error);
3092 }
3093
3094 static void
vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue * txq,bus_dmamap_t dmap)3095 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
3096 {
3097
3098 bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap);
3099 }
3100
3101 static int
vmxnet3_txq_encap(struct vmxnet3_txqueue * txq,struct mbuf ** m0)3102 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
3103 {
3104 struct vmxnet3_softc *sc;
3105 struct vmxnet3_txring *txr;
3106 struct vmxnet3_txdesc *txd, *sop;
3107 struct mbuf *m;
3108 bus_dmamap_t dmap;
3109 bus_dma_segment_t *segs;
3110 int i, gen, start, csum_start, nsegs, error;
3111
3112 sc = txq->vxtxq_sc;
3113 start = 0;
3114 txd = NULL;
3115 txr = &txq->vxtxq_cmd_ring;
3116 dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
3117 csum_start = 0; /* GCC */
3118
3119 error = vmxnet3_txq_load_mbuf(txq, m0, dmap);
3120 if (error)
3121 return (error);
3122
3123 nsegs = dmap->dm_nsegs;
3124 segs = dmap->dm_segs;
3125
3126 m = *m0;
3127 KASSERT(m->m_flags & M_PKTHDR);
3128 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS);
3129
3130 if (vmxnet3_txring_avail(txr) < nsegs) {
3131 txq->vxtxq_stats.vmtxs_full++;
3132 vmxnet3_txq_unload_mbuf(txq, dmap);
3133 return (ENOSPC);
3134 } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
3135 error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start);
3136 if (error) {
3137 /* m is already freed */
3138 txq->vxtxq_stats.vmtxs_offload_failed++;
3139 vmxnet3_txq_unload_mbuf(txq, dmap);
3140 *m0 = NULL;
3141 return (error);
3142 }
3143 }
3144
3145 txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
3146 sop = &txr->vxtxr_txd[txr->vxtxr_head];
3147 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */
3148
3149 for (i = 0; i < nsegs; i++) {
3150 txd = &txr->vxtxr_txd[txr->vxtxr_head];
3151
3152 txd->addr = segs[i].ds_addr;
3153 txd->len = segs[i].ds_len;
3154 txd->gen = gen;
3155 txd->dtype = 0;
3156 txd->offload_mode = VMXNET3_OM_NONE;
3157 txd->offload_pos = 0;
3158 txd->hlen = 0;
3159 txd->eop = 0;
3160 txd->compreq = 0;
3161 txd->vtag_mode = 0;
3162 txd->vtag = 0;
3163
3164 if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
3165 txr->vxtxr_head = 0;
3166 txr->vxtxr_gen ^= 1;
3167 }
3168 gen = txr->vxtxr_gen;
3169 }
3170 txd->eop = 1;
3171 txd->compreq = 1;
3172
3173 if (vlan_has_tag(m)) {
3174 sop->vtag_mode = 1;
3175 sop->vtag = vlan_get_tag(m);
3176 }
3177
3178 if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
3179 sop->offload_mode = VMXNET3_OM_TSO;
3180 sop->hlen = start;
3181 sop->offload_pos = m->m_pkthdr.segsz;
3182 } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
3183 VMXNET3_CSUM_OFFLOAD_IPV6)) {
3184 sop->offload_mode = VMXNET3_OM_CSUM;
3185 sop->hlen = start;
3186 sop->offload_pos = csum_start;
3187 }
3188
3189 /* Finally, change the ownership. */
3190 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
3191 sop->gen ^= 1;
3192
3193 txq->vxtxq_ts->npending += nsegs;
3194 if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
3195 struct vmxnet3_queue *vmxq;
3196 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue);
3197 txq->vxtxq_ts->npending = 0;
3198 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(vmxq->vxq_id),
3199 txr->vxtxr_head);
3200 }
3201
3202 return (0);
3203 }
3204
3205 #define VMXNET3_TX_START 1
3206 #define VMXNET3_TX_TRANSMIT 2
3207 static inline void
vmxnet3_tx_common_locked(struct ifnet * ifp,struct vmxnet3_txqueue * txq,int txtype)3208 vmxnet3_tx_common_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq, int txtype)
3209 {
3210 struct vmxnet3_softc *sc;
3211 struct vmxnet3_txring *txr;
3212 struct mbuf *m_head;
3213 int tx;
3214
3215 sc = ifp->if_softc;
3216 txr = &txq->vxtxq_cmd_ring;
3217 tx = 0;
3218
3219 VMXNET3_TXQ_LOCK_ASSERT(txq);
3220
3221 if (txq->vxtxq_stopping || sc->vmx_link_active == 0)
3222 return;
3223
3224 for (;;) {
3225 if (txtype == VMXNET3_TX_START)
3226 IFQ_POLL(&ifp->if_snd, m_head);
3227 else
3228 m_head = pcq_peek(txq->vxtxq_interq);
3229 if (m_head == NULL)
3230 break;
3231
3232 if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS)
3233 break;
3234
3235 if (txtype == VMXNET3_TX_START)
3236 IFQ_DEQUEUE(&ifp->if_snd, m_head);
3237 else
3238 m_head = pcq_get(txq->vxtxq_interq);
3239 if (m_head == NULL)
3240 break;
3241
3242 if (vmxnet3_txq_encap(txq, &m_head) != 0) {
3243 m_freem(m_head);
3244 break;
3245 }
3246
3247 tx++;
3248 bpf_mtap(ifp, m_head, BPF_D_OUT);
3249 }
3250
3251 if (tx > 0)
3252 txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
3253 }
3254
3255 static void
vmxnet3_start_locked(struct ifnet * ifp)3256 vmxnet3_start_locked(struct ifnet *ifp)
3257 {
3258 struct vmxnet3_softc *sc;
3259 struct vmxnet3_txqueue *txq;
3260
3261 sc = ifp->if_softc;
3262 txq = &sc->vmx_queue[0].vxq_txqueue;
3263
3264 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_START);
3265 }
3266
3267 void
vmxnet3_start(struct ifnet * ifp)3268 vmxnet3_start(struct ifnet *ifp)
3269 {
3270 struct vmxnet3_softc *sc;
3271 struct vmxnet3_txqueue *txq;
3272
3273 sc = ifp->if_softc;
3274 txq = &sc->vmx_queue[0].vxq_txqueue;
3275
3276 VMXNET3_TXQ_LOCK(txq);
3277 vmxnet3_start_locked(ifp);
3278 VMXNET3_TXQ_UNLOCK(txq);
3279 }
3280
3281 static int
vmxnet3_select_txqueue(struct ifnet * ifp,struct mbuf * m __unused)3282 vmxnet3_select_txqueue(struct ifnet *ifp, struct mbuf *m __unused)
3283 {
3284 struct vmxnet3_softc *sc;
3285 u_int cpuid;
3286
3287 sc = ifp->if_softc;
3288 cpuid = cpu_index(curcpu());
3289 /*
3290 * Future work
3291 * We should select txqueue to even up the load even if ncpu is
3292 * different from sc->vmx_ntxqueues. Currently, the load is not
3293 * even, that is, when ncpu is six and ntxqueues is four, the load
3294 * of vmx_queue[0] and vmx_queue[1] is higher than vmx_queue[2] and
3295 * vmx_queue[3] because CPU#4 always uses vmx_queue[0] and CPU#5 always
3296 * uses vmx_queue[1].
3297 * Furthermore, we should not use random value to select txqueue to
3298 * avoid reordering. We should use flow information of mbuf.
3299 */
3300 return cpuid % sc->vmx_ntxqueues;
3301 }
3302
3303 static void
vmxnet3_transmit_locked(struct ifnet * ifp,struct vmxnet3_txqueue * txq)3304 vmxnet3_transmit_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq)
3305 {
3306
3307 vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_TRANSMIT);
3308 }
3309
3310 static int
vmxnet3_transmit(struct ifnet * ifp,struct mbuf * m)3311 vmxnet3_transmit(struct ifnet *ifp, struct mbuf *m)
3312 {
3313 struct vmxnet3_softc *sc;
3314 struct vmxnet3_txqueue *txq;
3315 int qid;
3316
3317 qid = vmxnet3_select_txqueue(ifp, m);
3318 sc = ifp->if_softc;
3319 txq = &sc->vmx_queue[qid].vxq_txqueue;
3320
3321 if (__predict_false(!pcq_put(txq->vxtxq_interq, m))) {
3322 VMXNET3_TXQ_LOCK(txq);
3323 txq->vxtxq_pcqdrop.ev_count++;
3324 VMXNET3_TXQ_UNLOCK(txq);
3325 m_freem(m);
3326 return ENOBUFS;
3327 }
3328
3329 #ifdef VMXNET3_ALWAYS_TXDEFER
3330 kpreempt_disable();
3331 softint_schedule(txq->vxtxq_si);
3332 kpreempt_enable();
3333 #else
3334 if (VMXNET3_TXQ_TRYLOCK(txq)) {
3335 vmxnet3_transmit_locked(ifp, txq);
3336 VMXNET3_TXQ_UNLOCK(txq);
3337 } else {
3338 kpreempt_disable();
3339 softint_schedule(txq->vxtxq_si);
3340 kpreempt_enable();
3341 }
3342 #endif
3343
3344 return 0;
3345 }
3346
3347 static void
vmxnet3_deferred_transmit(void * arg)3348 vmxnet3_deferred_transmit(void *arg)
3349 {
3350 struct vmxnet3_txqueue *txq = arg;
3351 struct vmxnet3_softc *sc = txq->vxtxq_sc;
3352 struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
3353
3354 VMXNET3_TXQ_LOCK(txq);
3355 txq->vxtxq_transmitdef.ev_count++;
3356 if (pcq_peek(txq->vxtxq_interq) != NULL)
3357 vmxnet3_transmit_locked(ifp, txq);
3358 VMXNET3_TXQ_UNLOCK(txq);
3359 }
3360
3361 static void
vmxnet3_set_rxfilter(struct vmxnet3_softc * sc)3362 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
3363 {
3364 struct ethercom *ec = &sc->vmx_ethercom;
3365 struct vmxnet3_driver_shared *ds = sc->vmx_ds;
3366 struct ether_multi *enm;
3367 struct ether_multistep step;
3368 u_int mode;
3369 uint8_t *p;
3370
3371 VMXNET3_CORE_LOCK_ASSERT(sc);
3372
3373 ds->mcast_tablelen = 0;
3374 ETHER_LOCK(ec);
3375 CLR(ec->ec_flags, ETHER_F_ALLMULTI);
3376 ETHER_UNLOCK(ec);
3377
3378 /*
3379 * Always accept broadcast frames.
3380 * Always accept frames destined to our station address.
3381 */
3382 mode = VMXNET3_RXMODE_BCAST | VMXNET3_RXMODE_UCAST;
3383
3384 ETHER_LOCK(ec);
3385 if (sc->vmx_promisc ||
3386 ec->ec_multicnt > VMXNET3_MULTICAST_MAX)
3387 goto allmulti;
3388
3389 p = sc->vmx_mcast;
3390 ETHER_FIRST_MULTI(step, ec, enm);
3391 while (enm != NULL) {
3392 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
3393 /*
3394 * We must listen to a range of multicast addresses.
3395 * For now, just accept all multicasts, rather than
3396 * trying to set only those filter bits needed to match
3397 * the range. (At this time, the only use of address
3398 * ranges is for IP multicast routing, for which the
3399 * range is big enough to require all bits set.)
3400 */
3401 goto allmulti;
3402 }
3403 memcpy(p, enm->enm_addrlo, ETHER_ADDR_LEN);
3404
3405 p += ETHER_ADDR_LEN;
3406
3407 ETHER_NEXT_MULTI(step, enm);
3408 }
3409
3410 if (ec->ec_multicnt > 0) {
3411 SET(mode, VMXNET3_RXMODE_MCAST);
3412 ds->mcast_tablelen = p - sc->vmx_mcast;
3413 }
3414 ETHER_UNLOCK(ec);
3415
3416 goto setit;
3417
3418 allmulti:
3419 SET(ec->ec_flags, ETHER_F_ALLMULTI);
3420 ETHER_UNLOCK(ec);
3421 SET(mode, (VMXNET3_RXMODE_ALLMULTI | VMXNET3_RXMODE_MCAST));
3422 if (sc->vmx_promisc)
3423 SET(mode, VMXNET3_RXMODE_PROMISC);
3424
3425 setit:
3426 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
3427 ds->rxmode = mode;
3428 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
3429 }
3430
3431 static int
vmxnet3_ioctl(struct ifnet * ifp,u_long cmd,void * data)3432 vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data)
3433 {
3434 struct vmxnet3_softc *sc = ifp->if_softc;
3435 struct ifreq *ifr = (struct ifreq *)data;
3436 int s, error = 0;
3437
3438 switch (cmd) {
3439 case SIOCADDMULTI:
3440 case SIOCDELMULTI:
3441 break;
3442 default:
3443 KASSERT(IFNET_LOCKED(ifp));
3444 }
3445
3446 switch (cmd) {
3447 case SIOCSIFMTU: {
3448 int nmtu = ifr->ifr_mtu;
3449
3450 if (nmtu < VMXNET3_MIN_MTU || nmtu > VMXNET3_MAX_MTU) {
3451 error = EINVAL;
3452 break;
3453 }
3454 if (ifp->if_mtu != (uint64_t)nmtu) {
3455 s = splnet();
3456 error = ether_ioctl(ifp, cmd, data);
3457 splx(s);
3458 if (error == ENETRESET)
3459 error = vmxnet3_init(ifp);
3460 }
3461 break;
3462 }
3463
3464 default:
3465 s = splnet();
3466 error = ether_ioctl(ifp, cmd, data);
3467 splx(s);
3468 }
3469
3470 if (error == ENETRESET) {
3471 VMXNET3_CORE_LOCK(sc);
3472 if (sc->vmx_mcastactive)
3473 vmxnet3_set_rxfilter(sc);
3474 VMXNET3_CORE_UNLOCK(sc);
3475 error = 0;
3476 }
3477
3478 return error;
3479 }
3480
3481 static int
vmxnet3_ifflags_cb(struct ethercom * ec)3482 vmxnet3_ifflags_cb(struct ethercom *ec)
3483 {
3484 struct ifnet *ifp = &ec->ec_if;
3485 struct vmxnet3_softc *sc = ifp->if_softc;
3486 int error = 0;
3487
3488 KASSERT(IFNET_LOCKED(ifp));
3489
3490 VMXNET3_CORE_LOCK(sc);
3491 const unsigned short changed = ifp->if_flags ^ sc->vmx_if_flags;
3492 if ((changed & ~(IFF_CANTCHANGE | IFF_DEBUG)) == 0) {
3493 sc->vmx_if_flags = ifp->if_flags;
3494 if (changed & IFF_PROMISC) {
3495 sc->vmx_promisc = ifp->if_flags & IFF_PROMISC;
3496 error = ENETRESET;
3497 }
3498 } else {
3499 error = ENETRESET;
3500 }
3501 VMXNET3_CORE_UNLOCK(sc);
3502
3503 vmxnet3_if_link_status(sc);
3504
3505 return error;
3506 }
3507
3508 static int
vmxnet3_watchdog(struct vmxnet3_txqueue * txq)3509 vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
3510 {
3511 struct vmxnet3_softc *sc;
3512 struct vmxnet3_queue *vmxq;
3513
3514 sc = txq->vxtxq_sc;
3515 vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue);
3516
3517 VMXNET3_TXQ_LOCK(txq);
3518 if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
3519 VMXNET3_TXQ_UNLOCK(txq);
3520 return (0);
3521 }
3522 txq->vxtxq_watchdogto.ev_count++;
3523 VMXNET3_TXQ_UNLOCK(txq);
3524
3525 device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n",
3526 vmxq->vxq_id);
3527 return (1);
3528 }
3529
3530 static void
vmxnet3_refresh_host_stats(struct vmxnet3_softc * sc)3531 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
3532 {
3533
3534 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
3535 }
3536
3537 static void
vmxnet3_tick(void * xsc)3538 vmxnet3_tick(void *xsc)
3539 {
3540 struct vmxnet3_softc *sc;
3541 int i, timedout;
3542
3543 sc = xsc;
3544 timedout = 0;
3545
3546 VMXNET3_CORE_LOCK(sc);
3547
3548 vmxnet3_refresh_host_stats(sc);
3549
3550 for (i = 0; i < sc->vmx_ntxqueues; i++)
3551 timedout |= vmxnet3_watchdog(&sc->vmx_queue[i].vxq_txqueue);
3552
3553 if (timedout != 0) {
3554 if (!sc->vmx_reset_pending) {
3555 sc->vmx_reset_pending = true;
3556 workqueue_enqueue(sc->vmx_reset_wq,
3557 &sc->vmx_reset_work, NULL);
3558 }
3559 } else {
3560 callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
3561 }
3562
3563 VMXNET3_CORE_UNLOCK(sc);
3564 }
3565
3566 static void
vmxnet3_reset_work(struct work * work,void * arg)3567 vmxnet3_reset_work(struct work *work, void *arg)
3568 {
3569 struct vmxnet3_softc *sc = arg;
3570 struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
3571
3572 VMXNET3_CORE_LOCK(sc);
3573 KASSERT(sc->vmx_reset_pending);
3574 sc->vmx_reset_pending = false;
3575 VMXNET3_CORE_UNLOCK(sc);
3576
3577 IFNET_LOCK(ifp);
3578 (void)vmxnet3_init(ifp);
3579 IFNET_UNLOCK(ifp);
3580 }
3581
3582 /*
3583 * update link state of ifnet and softc
3584 */
3585 static void
vmxnet3_if_link_status(struct vmxnet3_softc * sc)3586 vmxnet3_if_link_status(struct vmxnet3_softc *sc)
3587 {
3588 struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
3589 u_int link;
3590 bool up;
3591
3592 up = vmxnet3_cmd_link_status(ifp);
3593 if (up) {
3594 sc->vmx_link_active = 1;
3595 link = LINK_STATE_UP;
3596 } else {
3597 sc->vmx_link_active = 0;
3598 link = LINK_STATE_DOWN;
3599 }
3600
3601 if_link_state_change(ifp, link);
3602 }
3603
3604 /*
3605 * check vmx(4) state by VMXNET3_CMD and update ifp->if_baudrate
3606 * returns
3607 * - true: link up
3608 * - false: link down
3609 */
3610 static bool
vmxnet3_cmd_link_status(struct ifnet * ifp)3611 vmxnet3_cmd_link_status(struct ifnet *ifp)
3612 {
3613 struct vmxnet3_softc *sc = ifp->if_softc;
3614 u_int x, speed;
3615
3616 x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
3617 if ((x & 1) == 0)
3618 return false;
3619
3620 speed = x >> 16;
3621 ifp->if_baudrate = IF_Mbps(speed);
3622 return true;
3623 }
3624
3625 static void
vmxnet3_ifmedia_status(struct ifnet * ifp,struct ifmediareq * ifmr)3626 vmxnet3_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3627 {
3628 bool up;
3629
3630 ifmr->ifm_status = IFM_AVALID;
3631 ifmr->ifm_active = IFM_ETHER;
3632
3633 up = vmxnet3_cmd_link_status(ifp);
3634 if (!up)
3635 return;
3636
3637 ifmr->ifm_status |= IFM_ACTIVE;
3638
3639 if (ifp->if_baudrate >= IF_Gbps(10ULL))
3640 ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
3641 }
3642
3643 static int
vmxnet3_ifmedia_change(struct ifnet * ifp)3644 vmxnet3_ifmedia_change(struct ifnet *ifp)
3645 {
3646 return 0;
3647 }
3648
3649 static void
vmxnet3_set_lladdr(struct vmxnet3_softc * sc)3650 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
3651 {
3652 uint32_t ml, mh;
3653
3654 ml = sc->vmx_lladdr[0];
3655 ml |= sc->vmx_lladdr[1] << 8;
3656 ml |= sc->vmx_lladdr[2] << 16;
3657 ml |= sc->vmx_lladdr[3] << 24;
3658 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
3659
3660 mh = sc->vmx_lladdr[4];
3661 mh |= sc->vmx_lladdr[5] << 8;
3662 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
3663 }
3664
3665 static void
vmxnet3_get_lladdr(struct vmxnet3_softc * sc)3666 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
3667 {
3668 uint32_t ml, mh;
3669
3670 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
3671 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
3672
3673 sc->vmx_lladdr[0] = ml;
3674 sc->vmx_lladdr[1] = ml >> 8;
3675 sc->vmx_lladdr[2] = ml >> 16;
3676 sc->vmx_lladdr[3] = ml >> 24;
3677 sc->vmx_lladdr[4] = mh;
3678 sc->vmx_lladdr[5] = mh >> 8;
3679 }
3680
3681 static void
vmxnet3_enable_all_intrs(struct vmxnet3_softc * sc)3682 vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
3683 {
3684 int i;
3685
3686 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
3687 for (i = 0; i < sc->vmx_nintrs; i++)
3688 vmxnet3_enable_intr(sc, i);
3689 }
3690
3691 static void
vmxnet3_disable_all_intrs(struct vmxnet3_softc * sc)3692 vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
3693 {
3694 int i;
3695
3696 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
3697 for (i = 0; i < sc->vmx_nintrs; i++)
3698 vmxnet3_disable_intr(sc, i);
3699 }
3700
3701 static int
vmxnet3_dma_malloc(struct vmxnet3_softc * sc,bus_size_t size,bus_size_t align,struct vmxnet3_dma_alloc * dma)3702 vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
3703 struct vmxnet3_dma_alloc *dma)
3704 {
3705 bus_dma_tag_t t = sc->vmx_dmat;
3706 bus_dma_segment_t *segs = dma->dma_segs;
3707 int n, error;
3708
3709 memset(dma, 0, sizeof(*dma));
3710
3711 error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT);
3712 if (error) {
3713 aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error);
3714 goto fail1;
3715 }
3716 KASSERT(n == 1);
3717
3718 error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT);
3719 if (error) {
3720 aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error);
3721 goto fail2;
3722 }
3723
3724 error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map);
3725 if (error) {
3726 aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error);
3727 goto fail3;
3728 }
3729
3730 error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL,
3731 BUS_DMA_NOWAIT);
3732 if (error) {
3733 aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error);
3734 goto fail4;
3735 }
3736
3737 memset(dma->dma_vaddr, 0, size);
3738 dma->dma_paddr = DMAADDR(dma->dma_map);
3739 dma->dma_size = size;
3740
3741 return (0);
3742 fail4:
3743 bus_dmamap_destroy(t, dma->dma_map);
3744 fail3:
3745 bus_dmamem_unmap(t, dma->dma_vaddr, size);
3746 fail2:
3747 bus_dmamem_free(t, segs, 1);
3748 fail1:
3749 return (error);
3750 }
3751
3752 static void
vmxnet3_dma_free(struct vmxnet3_softc * sc,struct vmxnet3_dma_alloc * dma)3753 vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
3754 {
3755 bus_dma_tag_t t = sc->vmx_dmat;
3756
3757 bus_dmamap_unload(t, dma->dma_map);
3758 bus_dmamap_destroy(t, dma->dma_map);
3759 bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size);
3760 bus_dmamem_free(t, dma->dma_segs, 1);
3761
3762 memset(dma, 0, sizeof(*dma));
3763 }
3764
3765 MODULE(MODULE_CLASS_DRIVER, if_vmx, "pci");
3766
3767 #ifdef _MODULE
3768 #include "ioconf.c"
3769 #endif
3770
3771 static int
if_vmx_modcmd(modcmd_t cmd,void * opaque)3772 if_vmx_modcmd(modcmd_t cmd, void *opaque)
3773 {
3774 int error = 0;
3775
3776 switch (cmd) {
3777 case MODULE_CMD_INIT:
3778 #ifdef _MODULE
3779 error = config_init_component(cfdriver_ioconf_if_vmx,
3780 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx);
3781 #endif
3782 return error;
3783 case MODULE_CMD_FINI:
3784 #ifdef _MODULE
3785 error = config_fini_component(cfdriver_ioconf_if_vmx,
3786 cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx);
3787 #endif
3788 return error;
3789 default:
3790 return ENOTTY;
3791 }
3792 }
3793
3794