1 /* $NetBSD: ixv.c,v 1.198 2024/07/10 03:26:30 msaitoh Exp $ */
2
3 /******************************************************************************
4
5 Copyright (c) 2001-2017, Intel Corporation
6 All rights reserved.
7
8 Redistribution and use in source and binary forms, with or without
9 modification, are permitted provided that the following conditions are met:
10
11 1. Redistributions of source code must retain the above copyright notice,
12 this list of conditions and the following disclaimer.
13
14 2. Redistributions in binary form must reproduce the above copyright
15 notice, this list of conditions and the following disclaimer in the
16 documentation and/or other materials provided with the distribution.
17
18 3. Neither the name of the Intel Corporation nor the names of its
19 contributors may be used to endorse or promote products derived from
20 this software without specific prior written permission.
21
22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 POSSIBILITY OF SUCH DAMAGE.
33
34 ******************************************************************************/
35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/
36
37 #include <sys/cdefs.h>
38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.198 2024/07/10 03:26:30 msaitoh Exp $");
39
40 #ifdef _KERNEL_OPT
41 #include "opt_inet.h"
42 #include "opt_inet6.h"
43 #endif
44
45 #include "ixgbe.h"
46
47 /************************************************************************
48 * Driver version
49 ************************************************************************/
50 static const char ixv_driver_version[] = "2.0.1-k";
51 /* XXX NetBSD: + 1.5.17 */
52
53 /************************************************************************
54 * PCI Device ID Table
55 *
56 * Used by probe to select devices to load on
57 * Last field stores an index into ixv_strings
58 * Last entry must be all 0s
59 *
60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61 ************************************************************************/
62 static const ixgbe_vendor_info_t ixv_vendor_info_array[] =
63 {
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0},
69 /* required last entry */
70 {0, 0, 0, 0, 0}
71 };
72
73 /************************************************************************
74 * Table of branding strings
75 ************************************************************************/
76 static const char *ixv_strings[] = {
77 "Intel(R) PRO/10GbE Virtual Function Network Driver"
78 };
79
80 /*********************************************************************
81 * Function prototypes
82 *********************************************************************/
83 static int ixv_probe(device_t, cfdata_t, void *);
84 static void ixv_attach(device_t, device_t, void *);
85 static int ixv_detach(device_t, int);
86 #if 0
87 static int ixv_shutdown(device_t);
88 #endif
89 static int ixv_ifflags_cb(struct ethercom *);
90 static int ixv_ioctl(struct ifnet *, u_long, void *);
91 static int ixv_init(struct ifnet *);
92 static void ixv_init_locked(struct ixgbe_softc *);
93 static void ixv_ifstop(struct ifnet *, int);
94 static void ixv_stop_locked(void *);
95 static void ixv_init_device_features(struct ixgbe_softc *);
96 static void ixv_media_status(struct ifnet *, struct ifmediareq *);
97 static int ixv_media_change(struct ifnet *);
98 static int ixv_allocate_pci_resources(struct ixgbe_softc *,
99 const struct pci_attach_args *);
100 static void ixv_free_deferred_handlers(struct ixgbe_softc *);
101 static int ixv_allocate_msix(struct ixgbe_softc *,
102 const struct pci_attach_args *);
103 static int ixv_configure_interrupts(struct ixgbe_softc *);
104 static void ixv_free_pci_resources(struct ixgbe_softc *);
105 static void ixv_local_timer(void *);
106 static void ixv_handle_timer(struct work *, void *);
107 static int ixv_setup_interface(device_t, struct ixgbe_softc *);
108 static void ixv_schedule_admin_tasklet(struct ixgbe_softc *);
109 static int ixv_negotiate_api(struct ixgbe_softc *);
110
111 static void ixv_initialize_transmit_units(struct ixgbe_softc *);
112 static void ixv_initialize_receive_units(struct ixgbe_softc *);
113 static void ixv_initialize_rss_mapping(struct ixgbe_softc *);
114 static s32 ixv_check_link(struct ixgbe_softc *);
115
116 static void ixv_enable_intr(struct ixgbe_softc *);
117 static void ixv_disable_intr(struct ixgbe_softc *);
118 static int ixv_set_rxfilter(struct ixgbe_softc *);
119 static void ixv_update_link_status(struct ixgbe_softc *);
120 static int ixv_sysctl_debug(SYSCTLFN_PROTO);
121 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8);
122 static void ixv_configure_ivars(struct ixgbe_softc *);
123 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
124 static void ixv_eitr_write(struct ixgbe_softc *, uint32_t, uint32_t);
125
126 static void ixv_setup_vlan_tagging(struct ixgbe_softc *);
127 static int ixv_setup_vlan_support(struct ixgbe_softc *);
128 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool);
129 static int ixv_register_vlan(struct ixgbe_softc *, u16);
130 static int ixv_unregister_vlan(struct ixgbe_softc *, u16);
131
132 static void ixv_add_device_sysctls(struct ixgbe_softc *);
133 static void ixv_init_stats(struct ixgbe_softc *);
134 static void ixv_update_stats(struct ixgbe_softc *);
135 static void ixv_add_stats_sysctls(struct ixgbe_softc *);
136 static void ixv_clear_evcnt(struct ixgbe_softc *);
137
138 /* Sysctl handlers */
139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO);
140 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO);
141 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO);
142 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO);
143 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO);
144 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO);
145 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO);
146 static int ixv_sysctl_tx_process_limit(SYSCTLFN_PROTO);
147 static int ixv_sysctl_rx_process_limit(SYSCTLFN_PROTO);
148 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO);
149
150 /* The MSI-X Interrupt handlers */
151 static int ixv_msix_que(void *);
152 static int ixv_msix_mbx(void *);
153
154 /* Event handlers running on workqueue */
155 static void ixv_handle_que(void *);
156
157 /* Deferred workqueue handlers */
158 static void ixv_handle_admin(struct work *, void *);
159 static void ixv_handle_que_work(struct work *, void *);
160
161 const struct sysctlnode *ixv_sysctl_instance(struct ixgbe_softc *);
162 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *);
163
164 /************************************************************************
165 * NetBSD Device Interface Entry Points
166 ************************************************************************/
167 CFATTACH_DECL3_NEW(ixv, sizeof(struct ixgbe_softc),
168 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL,
169 DVF_DETACH_SHUTDOWN);
170
171 #if 0
172 static driver_t ixv_driver = {
173 "ixv", ixv_methods, sizeof(struct ixgbe_softc),
174 };
175
176 devclass_t ixv_devclass;
177 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0);
178 MODULE_DEPEND(ixv, pci, 1, 1, 1);
179 MODULE_DEPEND(ixv, ether, 1, 1, 1);
180 #endif
181
182 /*
183 * TUNEABLE PARAMETERS:
184 */
185
186 /* Number of Queues - do not exceed MSI-X vectors - 1 */
187 static int ixv_num_queues = 0;
188 #define TUNABLE_INT(__x, __y)
189 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues);
190
191 /*
192 * AIM: Adaptive Interrupt Moderation
193 * which means that the interrupt rate
194 * is varied over time based on the
195 * traffic for that interrupt vector
196 */
197 static bool ixv_enable_aim = false;
198 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim);
199
200 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
201 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate);
202
203 /* How many packets rxeof tries to clean at a time */
204 static int ixv_rx_process_limit = 256;
205 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit);
206
207 /* How many packets txeof tries to clean at a time */
208 static int ixv_tx_process_limit = 256;
209 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit);
210
211 /* Which packet processing uses workqueue or softint */
212 static bool ixv_txrx_workqueue = false;
213
214 /*
215 * Number of TX descriptors per ring,
216 * setting higher than RX as this seems
217 * the better performing choice.
218 */
219 static int ixv_txd = DEFAULT_TXD;
220 TUNABLE_INT("hw.ixv.txd", &ixv_txd);
221
222 /* Number of RX descriptors per ring */
223 static int ixv_rxd = DEFAULT_RXD;
224 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd);
225
226 /* Legacy Transmit (single queue) */
227 static int ixv_enable_legacy_tx = 0;
228 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx);
229
230 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET
231
232 #if 0
233 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *);
234 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *);
235 #endif
236
237 /************************************************************************
238 * ixv_probe - Device identification routine
239 *
240 * Determines if the driver should be loaded on
241 * adapter based on its PCI vendor/device ID.
242 *
243 * return BUS_PROBE_DEFAULT on success, positive on failure
244 ************************************************************************/
245 static int
ixv_probe(device_t dev,cfdata_t cf,void * aux)246 ixv_probe(device_t dev, cfdata_t cf, void *aux)
247 {
248 #ifdef __HAVE_PCI_MSI_MSIX
249 const struct pci_attach_args *pa = aux;
250
251 return (ixv_lookup(pa) != NULL) ? 1 : 0;
252 #else
253 return 0;
254 #endif
255 } /* ixv_probe */
256
257 static const ixgbe_vendor_info_t *
ixv_lookup(const struct pci_attach_args * pa)258 ixv_lookup(const struct pci_attach_args *pa)
259 {
260 const ixgbe_vendor_info_t *ent;
261 pcireg_t subid;
262
263 INIT_DEBUGOUT("ixv_lookup: begin");
264
265 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID)
266 return NULL;
267
268 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
269
270 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) {
271 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) &&
272 (PCI_PRODUCT(pa->pa_id) == ent->device_id) &&
273 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) ||
274 (ent->subvendor_id == 0)) &&
275 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) ||
276 (ent->subdevice_id == 0))) {
277 return ent;
278 }
279 }
280
281 return NULL;
282 }
283
284 /************************************************************************
285 * ixv_attach - Device initialization routine
286 *
287 * Called when the driver is being loaded.
288 * Identifies the type of hardware, allocates all resources
289 * and initializes the hardware.
290 *
291 * return 0 on success, positive on failure
292 ************************************************************************/
293 static void
ixv_attach(device_t parent,device_t dev,void * aux)294 ixv_attach(device_t parent, device_t dev, void *aux)
295 {
296 struct ixgbe_softc *sc;
297 struct ixgbe_hw *hw;
298 int error = 0;
299 pcireg_t id, subid;
300 const ixgbe_vendor_info_t *ent;
301 const struct pci_attach_args *pa = aux;
302 const char *apivstr;
303 const char *str;
304 char wqname[MAXCOMLEN];
305 char buf[256];
306
307 INIT_DEBUGOUT("ixv_attach: begin");
308
309 /*
310 * Make sure BUSMASTER is set, on a VM under
311 * KVM it may not be and will break things.
312 */
313 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag);
314
315 /* Allocate, clear, and link in our adapter structure */
316 sc = device_private(dev);
317 sc->hw.back = sc;
318 sc->dev = dev;
319 hw = &sc->hw;
320
321 sc->init_locked = ixv_init_locked;
322 sc->stop_locked = ixv_stop_locked;
323
324 sc->osdep.pc = pa->pa_pc;
325 sc->osdep.tag = pa->pa_tag;
326 if (pci_dma64_available(pa))
327 sc->osdep.dmat = pa->pa_dmat64;
328 else
329 sc->osdep.dmat = pa->pa_dmat;
330 sc->osdep.attached = false;
331
332 ent = ixv_lookup(pa);
333
334 KASSERT(ent != NULL);
335
336 aprint_normal(": %s, Version - %s\n",
337 ixv_strings[ent->index], ixv_driver_version);
338
339 /* Core Lock Init */
340 IXGBE_CORE_LOCK_INIT(sc, device_xname(dev));
341
342 /* Do base PCI setup - map BAR0 */
343 if (ixv_allocate_pci_resources(sc, pa)) {
344 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n");
345 error = ENXIO;
346 goto err_out;
347 }
348
349 /* SYSCTL APIs */
350 ixv_add_device_sysctls(sc);
351
352 /* Set up the timer callout and workqueue */
353 callout_init(&sc->timer, CALLOUT_MPSAFE);
354 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev));
355 error = workqueue_create(&sc->timer_wq, wqname,
356 ixv_handle_timer, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE);
357 if (error) {
358 aprint_error_dev(dev,
359 "could not create timer workqueue (%d)\n", error);
360 goto err_out;
361 }
362
363 /* Save off the information about this board */
364 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG);
365 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
366 hw->vendor_id = PCI_VENDOR(id);
367 hw->device_id = PCI_PRODUCT(id);
368 hw->revision_id =
369 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG));
370 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid);
371 hw->subsystem_device_id = PCI_SUBSYS_ID(subid);
372
373 /* A subset of set_mac_type */
374 switch (hw->device_id) {
375 case IXGBE_DEV_ID_82599_VF:
376 hw->mac.type = ixgbe_mac_82599_vf;
377 str = "82599 VF";
378 break;
379 case IXGBE_DEV_ID_X540_VF:
380 hw->mac.type = ixgbe_mac_X540_vf;
381 str = "X540 VF";
382 break;
383 case IXGBE_DEV_ID_X550_VF:
384 hw->mac.type = ixgbe_mac_X550_vf;
385 str = "X550 VF";
386 break;
387 case IXGBE_DEV_ID_X550EM_X_VF:
388 hw->mac.type = ixgbe_mac_X550EM_x_vf;
389 str = "X550EM X VF";
390 break;
391 case IXGBE_DEV_ID_X550EM_A_VF:
392 hw->mac.type = ixgbe_mac_X550EM_a_vf;
393 str = "X550EM A VF";
394 break;
395 default:
396 /* Shouldn't get here since probe succeeded */
397 aprint_error_dev(dev, "Unknown device ID!\n");
398 error = ENXIO;
399 goto err_out;
400 break;
401 }
402 aprint_normal_dev(dev, "device %s\n", str);
403
404 ixv_init_device_features(sc);
405
406 /* Initialize the shared code */
407 error = ixgbe_init_ops_vf(hw);
408 if (error) {
409 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n");
410 error = EIO;
411 goto err_out;
412 }
413
414 /* Setup the mailbox */
415 ixgbe_init_mbx_params_vf(hw);
416
417 /* Set the right number of segments */
418 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT);
419 sc->num_segs = IXGBE_SCATTER_DEFAULT;
420
421 /* Reset mbox api to 1.0 */
422 error = hw->mac.ops.reset_hw(hw);
423 if (error == IXGBE_ERR_RESET_FAILED)
424 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n");
425 else if (error)
426 aprint_error_dev(dev, "...reset_hw() failed with error %d\n",
427 error);
428 if (error) {
429 error = EIO;
430 goto err_out;
431 }
432
433 error = hw->mac.ops.init_hw(hw);
434 if (error) {
435 aprint_error_dev(dev, "...init_hw() failed!\n");
436 error = EIO;
437 goto err_out;
438 }
439
440 /* Negotiate mailbox API version */
441 error = ixv_negotiate_api(sc);
442 if (error)
443 aprint_normal_dev(dev,
444 "MBX API negotiation failed during attach!\n");
445 switch (hw->api_version) {
446 case ixgbe_mbox_api_10:
447 apivstr = "1.0";
448 break;
449 case ixgbe_mbox_api_20:
450 apivstr = "2.0";
451 break;
452 case ixgbe_mbox_api_11:
453 apivstr = "1.1";
454 break;
455 case ixgbe_mbox_api_12:
456 apivstr = "1.2";
457 break;
458 case ixgbe_mbox_api_13:
459 apivstr = "1.3";
460 break;
461 case ixgbe_mbox_api_14:
462 apivstr = "1.4";
463 break;
464 case ixgbe_mbox_api_15:
465 apivstr = "1.5";
466 break;
467 default:
468 apivstr = "unknown";
469 break;
470 }
471 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr);
472
473 /* If no mac address was assigned, make a random one */
474 if (!ixv_check_ether_addr(hw->mac.addr)) {
475 u8 addr[ETHER_ADDR_LEN];
476 uint64_t rndval = cprng_strong64();
477
478 memcpy(addr, &rndval, sizeof(addr));
479 addr[0] &= 0xFE;
480 addr[0] |= 0x02;
481 bcopy(addr, hw->mac.addr, sizeof(addr));
482 }
483
484 /* Register for VLAN events */
485 ether_set_vlan_cb(&sc->osdep.ec, ixv_vlan_cb);
486
487 /* Do descriptor calc and sanity checks */
488 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
489 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) {
490 aprint_error_dev(dev, "Invalid TX ring size (%d). "
491 "It must be between %d and %d, "
492 "inclusive, and must be a multiple of %zu. "
493 "Using default value of %d instead.\n",
494 ixv_txd, MIN_TXD, MAX_TXD,
495 DBA_ALIGN / sizeof(union ixgbe_adv_tx_desc),
496 DEFAULT_TXD);
497 sc->num_tx_desc = DEFAULT_TXD;
498 } else
499 sc->num_tx_desc = ixv_txd;
500
501 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
502 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) {
503 aprint_error_dev(dev, "Invalid RX ring size (%d). "
504 "It must be between %d and %d, "
505 "inclusive, and must be a multiple of %zu. "
506 "Using default value of %d instead.\n",
507 ixv_rxd, MIN_RXD, MAX_RXD,
508 DBA_ALIGN / sizeof(union ixgbe_adv_rx_desc),
509 DEFAULT_RXD);
510 sc->num_rx_desc = DEFAULT_RXD;
511 } else
512 sc->num_rx_desc = ixv_rxd;
513
514 /* Sysctls for limiting the amount of work done in the taskqueues */
515 sc->rx_process_limit
516 = (ixv_rx_process_limit <= sc->num_rx_desc)
517 ? ixv_rx_process_limit : sc->num_rx_desc;
518 sc->tx_process_limit
519 = (ixv_tx_process_limit <= sc->num_tx_desc)
520 ? ixv_tx_process_limit : sc->num_tx_desc;
521
522 /* Set default high limit of copying mbuf in rxeof */
523 sc->rx_copy_len = IXGBE_RX_COPY_LEN_MAX;
524
525 /* Setup MSI-X */
526 error = ixv_configure_interrupts(sc);
527 if (error)
528 goto err_out;
529
530 /* Allocate our TX/RX Queues */
531 if (ixgbe_allocate_queues(sc)) {
532 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n");
533 error = ENOMEM;
534 goto err_out;
535 }
536
537 /* hw.ix defaults init */
538 sc->enable_aim = ixv_enable_aim;
539 sc->max_interrupt_rate = ixv_max_interrupt_rate;
540
541 sc->txrx_use_workqueue = ixv_txrx_workqueue;
542
543 error = ixv_allocate_msix(sc, pa);
544 if (error) {
545 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n");
546 goto err_late;
547 }
548
549 /* Setup OS specific network interface */
550 error = ixv_setup_interface(dev, sc);
551 if (error != 0) {
552 aprint_error_dev(dev, "ixv_setup_interface() failed!\n");
553 goto err_late;
554 }
555
556 /* Allocate multicast array memory */
557 sc->mta = malloc(sizeof(*sc->mta) *
558 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK);
559
560 /* Check if VF was disabled by PF */
561 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled);
562 if (error) {
563 /* PF is not capable of controlling VF state. Enable the link. */
564 sc->link_enabled = TRUE;
565 }
566
567 /* Do the stats setup */
568 ixv_init_stats(sc);
569 ixv_add_stats_sysctls(sc);
570
571 if (sc->feat_en & IXGBE_FEATURE_NETMAP)
572 ixgbe_netmap_attach(sc);
573
574 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_cap);
575 aprint_verbose_dev(dev, "feature cap %s\n", buf);
576 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_en);
577 aprint_verbose_dev(dev, "feature ena %s\n", buf);
578
579 INIT_DEBUGOUT("ixv_attach: end");
580 sc->osdep.attached = true;
581
582 return;
583
584 err_late:
585 ixgbe_free_queues(sc);
586 err_out:
587 ixv_free_pci_resources(sc);
588 IXGBE_CORE_LOCK_DESTROY(sc);
589
590 return;
591 } /* ixv_attach */
592
593 /************************************************************************
594 * ixv_detach - Device removal routine
595 *
596 * Called when the driver is being removed.
597 * Stops the adapter and deallocates all the resources
598 * that were allocated for driver operation.
599 *
600 * return 0 on success, positive on failure
601 ************************************************************************/
602 static int
ixv_detach(device_t dev,int flags)603 ixv_detach(device_t dev, int flags)
604 {
605 struct ixgbe_softc *sc = device_private(dev);
606 struct ixgbe_hw *hw = &sc->hw;
607 struct tx_ring *txr = sc->tx_rings;
608 struct rx_ring *rxr = sc->rx_rings;
609 struct ixgbevf_hw_stats *stats = &sc->stats.vf;
610
611 INIT_DEBUGOUT("ixv_detach: begin");
612 if (sc->osdep.attached == false)
613 return 0;
614
615 /* Stop the interface. Callouts are stopped in it. */
616 ixv_ifstop(sc->ifp, 1);
617
618 if (VLAN_ATTACHED(&sc->osdep.ec) &&
619 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) {
620 aprint_error_dev(dev, "VLANs in use, detach first\n");
621 return EBUSY;
622 }
623
624 ether_ifdetach(sc->ifp);
625 callout_halt(&sc->timer, NULL);
626 ixv_free_deferred_handlers(sc);
627
628 if (sc->feat_en & IXGBE_FEATURE_NETMAP)
629 netmap_detach(sc->ifp);
630
631 ixv_free_pci_resources(sc);
632 #if 0 /* XXX the NetBSD port is probably missing something here */
633 bus_generic_detach(dev);
634 #endif
635 if_detach(sc->ifp);
636 ifmedia_fini(&sc->media);
637 if_percpuq_destroy(sc->ipq);
638
639 sysctl_teardown(&sc->sysctllog);
640 evcnt_detach(&sc->efbig_tx_dma_setup);
641 evcnt_detach(&sc->mbuf_defrag_failed);
642 evcnt_detach(&sc->efbig2_tx_dma_setup);
643 evcnt_detach(&sc->einval_tx_dma_setup);
644 evcnt_detach(&sc->other_tx_dma_setup);
645 evcnt_detach(&sc->eagain_tx_dma_setup);
646 evcnt_detach(&sc->enomem_tx_dma_setup);
647 evcnt_detach(&sc->watchdog_events);
648 evcnt_detach(&sc->tso_err);
649 evcnt_detach(&sc->admin_irqev);
650 evcnt_detach(&sc->link_workev);
651
652 txr = sc->tx_rings;
653 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) {
654 evcnt_detach(&sc->queues[i].irqs);
655 evcnt_detach(&sc->queues[i].handleq);
656 evcnt_detach(&sc->queues[i].req);
657 evcnt_detach(&txr->total_packets);
658 #ifndef IXGBE_LEGACY_TX
659 evcnt_detach(&txr->pcq_drops);
660 #endif
661 evcnt_detach(&txr->no_desc_avail);
662 evcnt_detach(&txr->tso_tx);
663
664 evcnt_detach(&rxr->rx_packets);
665 evcnt_detach(&rxr->rx_bytes);
666 evcnt_detach(&rxr->rx_copies);
667 evcnt_detach(&rxr->no_mbuf);
668 evcnt_detach(&rxr->rx_discarded);
669 }
670 evcnt_detach(&stats->ipcs);
671 evcnt_detach(&stats->l4cs);
672 evcnt_detach(&stats->ipcs_bad);
673 evcnt_detach(&stats->l4cs_bad);
674
675 /* Packet Reception Stats */
676 evcnt_detach(&stats->vfgorc);
677 evcnt_detach(&stats->vfgprc);
678 evcnt_detach(&stats->vfmprc);
679
680 /* Packet Transmission Stats */
681 evcnt_detach(&stats->vfgotc);
682 evcnt_detach(&stats->vfgptc);
683
684 /* Mailbox Stats */
685 evcnt_detach(&hw->mbx.stats.msgs_tx);
686 evcnt_detach(&hw->mbx.stats.msgs_rx);
687 evcnt_detach(&hw->mbx.stats.acks);
688 evcnt_detach(&hw->mbx.stats.reqs);
689 evcnt_detach(&hw->mbx.stats.rsts);
690
691 ixgbe_free_queues(sc);
692
693 IXGBE_CORE_LOCK_DESTROY(sc);
694
695 return (0);
696 } /* ixv_detach */
697
698 /************************************************************************
699 * ixv_init_locked - Init entry point
700 *
701 * Used in two ways: It is used by the stack as an init entry
702 * point in network interface structure. It is also used
703 * by the driver as a hw/sw initialization routine to get
704 * to a consistent state.
705 *
706 * return 0 on success, positive on failure
707 ************************************************************************/
708 static void
ixv_init_locked(struct ixgbe_softc * sc)709 ixv_init_locked(struct ixgbe_softc *sc)
710 {
711 struct ifnet *ifp = sc->ifp;
712 device_t dev = sc->dev;
713 struct ixgbe_hw *hw = &sc->hw;
714 struct ix_queue *que;
715 int error = 0;
716 uint32_t mask;
717 int i;
718
719 INIT_DEBUGOUT("ixv_init_locked: begin");
720 KASSERT(mutex_owned(&sc->core_mtx));
721 hw->adapter_stopped = FALSE;
722 hw->mac.ops.stop_adapter(hw);
723 callout_stop(&sc->timer);
724 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++)
725 que->disabled_count = 0;
726
727 sc->max_frame_size =
728 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
729
730 /* reprogram the RAR[0] in case user changed it. */
731 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
732
733 /* Get the latest mac address, User can use a LAA */
734 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl),
735 IXGBE_ETH_LENGTH_OF_ADDRESS);
736 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1);
737
738 /* Prepare transmit descriptors and buffers */
739 if (ixgbe_setup_transmit_structures(sc)) {
740 aprint_error_dev(dev, "Could not setup transmit structures\n");
741 ixv_stop_locked(sc);
742 return;
743 }
744
745 /* Reset VF and renegotiate mailbox API version */
746 hw->mac.ops.reset_hw(hw);
747 hw->mac.ops.start_hw(hw);
748 error = ixv_negotiate_api(sc);
749 if (error)
750 device_printf(dev,
751 "Mailbox API negotiation failed in init_locked!\n");
752
753 ixv_initialize_transmit_units(sc);
754
755 /* Setup Multicast table */
756 ixv_set_rxfilter(sc);
757
758 /* Use fixed buffer size, even for jumbo frames */
759 sc->rx_mbuf_sz = MCLBYTES;
760
761 /* Prepare receive descriptors and buffers */
762 error = ixgbe_setup_receive_structures(sc);
763 if (error) {
764 device_printf(dev,
765 "Could not setup receive structures (err = %d)\n", error);
766 ixv_stop_locked(sc);
767 return;
768 }
769
770 /* Configure RX settings */
771 ixv_initialize_receive_units(sc);
772
773 /* Initialize variable holding task enqueue requests interrupts */
774 sc->task_requests = 0;
775
776 /* Set up VLAN offload and filter */
777 ixv_setup_vlan_support(sc);
778
779 /* Set up MSI-X routing */
780 ixv_configure_ivars(sc);
781
782 /* Set up auto-mask */
783 mask = (1 << sc->vector);
784 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++)
785 mask |= (1 << que->msix);
786 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask);
787
788 /* Set moderation on the Link interrupt */
789 ixv_eitr_write(sc, sc->vector, IXGBE_LINK_ITR);
790
791 /* Stats init */
792 ixv_init_stats(sc);
793
794 /* Config/Enable Link */
795 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled);
796 if (error) {
797 /* PF is not capable of controlling VF state. Enable the link. */
798 sc->link_enabled = TRUE;
799 } else if (sc->link_enabled == FALSE)
800 device_printf(dev, "VF is disabled by PF\n");
801
802 hw->mac.get_link_status = TRUE;
803 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up,
804 FALSE);
805
806 /* Start watchdog */
807 callout_reset(&sc->timer, hz, ixv_local_timer, sc);
808 atomic_store_relaxed(&sc->timer_pending, 0);
809
810 /* OK to schedule workqueues. */
811 sc->schedule_wqs_ok = true;
812
813 /* Update saved flags. See ixgbe_ifflags_cb() */
814 sc->if_flags = ifp->if_flags;
815 sc->ec_capenable = sc->osdep.ec.ec_capenable;
816
817 /* Inform the stack we're ready */
818 ifp->if_flags |= IFF_RUNNING;
819
820 /* And now turn on interrupts */
821 ixv_enable_intr(sc);
822
823 return;
824 } /* ixv_init_locked */
825
826 /************************************************************************
827 * ixv_enable_queue
828 ************************************************************************/
829 static inline void
ixv_enable_queue(struct ixgbe_softc * sc,u32 vector)830 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector)
831 {
832 struct ixgbe_hw *hw = &sc->hw;
833 struct ix_queue *que = &sc->queues[vector];
834 u32 queue = 1UL << vector;
835 u32 mask;
836
837 mutex_enter(&que->dc_mtx);
838 if (que->disabled_count > 0 && --que->disabled_count > 0)
839 goto out;
840
841 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
842 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
843 out:
844 mutex_exit(&que->dc_mtx);
845 } /* ixv_enable_queue */
846
847 /************************************************************************
848 * ixv_disable_queue
849 ************************************************************************/
850 static inline void
ixv_disable_queue(struct ixgbe_softc * sc,u32 vector)851 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector)
852 {
853 struct ixgbe_hw *hw = &sc->hw;
854 struct ix_queue *que = &sc->queues[vector];
855 u32 queue = 1UL << vector;
856 u32 mask;
857
858 mutex_enter(&que->dc_mtx);
859 if (que->disabled_count++ > 0)
860 goto out;
861
862 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
863 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask);
864 out:
865 mutex_exit(&que->dc_mtx);
866 } /* ixv_disable_queue */
867
868 #if 0
869 static inline void
870 ixv_rearm_queues(struct ixgbe_softc *sc, u64 queues)
871 {
872 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
873 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEICS, mask);
874 } /* ixv_rearm_queues */
875 #endif
876
877
878 /************************************************************************
879 * ixv_msix_que - MSI-X Queue Interrupt Service routine
880 ************************************************************************/
881 static int
ixv_msix_que(void * arg)882 ixv_msix_que(void *arg)
883 {
884 struct ix_queue *que = arg;
885 struct ixgbe_softc *sc = que->sc;
886 struct tx_ring *txr = que->txr;
887 struct rx_ring *rxr = que->rxr;
888 bool more;
889 u32 newitr = 0;
890
891 ixv_disable_queue(sc, que->msix);
892 IXGBE_EVC_ADD(&que->irqs, 1);
893
894 #ifdef __NetBSD__
895 /* Don't run ixgbe_rxeof in interrupt context */
896 more = true;
897 #else
898 more = ixgbe_rxeof(que);
899 #endif
900
901 IXGBE_TX_LOCK(txr);
902 ixgbe_txeof(txr);
903 IXGBE_TX_UNLOCK(txr);
904
905 /* Do AIM now? */
906
907 if (sc->enable_aim == false)
908 goto no_calc;
909 /*
910 * Do Adaptive Interrupt Moderation:
911 * - Write out last calculated setting
912 * - Calculate based on average size over
913 * the last interval.
914 */
915 if (que->eitr_setting)
916 ixv_eitr_write(sc, que->msix, que->eitr_setting);
917
918 que->eitr_setting = 0;
919
920 /* Idle, do nothing */
921 if ((txr->bytes == 0) && (rxr->bytes == 0))
922 goto no_calc;
923
924 if ((txr->bytes) && (txr->packets))
925 newitr = txr->bytes/txr->packets;
926 if ((rxr->bytes) && (rxr->packets))
927 newitr = uimax(newitr, (rxr->bytes / rxr->packets));
928 newitr += 24; /* account for hardware frame, crc */
929
930 /* set an upper boundary */
931 newitr = uimin(newitr, 3000);
932
933 /* Be nice to the mid range */
934 if ((newitr > 300) && (newitr < 1200))
935 newitr = (newitr / 3);
936 else
937 newitr = (newitr / 2);
938
939 /*
940 * When RSC is used, ITR interval must be larger than RSC_DELAY.
941 * Currently, we use 2us for RSC_DELAY. The minimum value is always
942 * greater than 2us on 100M (and 10M?(not documented)), but it's not
943 * on 1G and higher.
944 */
945 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL)
946 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) {
947 if (newitr < IXGBE_MIN_RSC_EITR_10G1G)
948 newitr = IXGBE_MIN_RSC_EITR_10G1G;
949 }
950
951 /* save for next interrupt */
952 que->eitr_setting = newitr;
953
954 /* Reset state */
955 txr->bytes = 0;
956 txr->packets = 0;
957 rxr->bytes = 0;
958 rxr->packets = 0;
959
960 no_calc:
961 if (more)
962 softint_schedule(que->que_si);
963 else /* Re-enable this interrupt */
964 ixv_enable_queue(sc, que->msix);
965
966 return 1;
967 } /* ixv_msix_que */
968
969 /************************************************************************
970 * ixv_msix_mbx
971 ************************************************************************/
972 static int
ixv_msix_mbx(void * arg)973 ixv_msix_mbx(void *arg)
974 {
975 struct ixgbe_softc *sc = arg;
976 struct ixgbe_hw *hw = &sc->hw;
977
978 IXGBE_EVC_ADD(&sc->admin_irqev, 1);
979 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */
980
981 /* Link status change */
982 hw->mac.get_link_status = TRUE;
983 atomic_or_32(&sc->task_requests, IXGBE_REQUEST_TASK_MBX);
984 ixv_schedule_admin_tasklet(sc);
985
986 return 1;
987 } /* ixv_msix_mbx */
988
989 static void
ixv_eitr_write(struct ixgbe_softc * sc,uint32_t index,uint32_t itr)990 ixv_eitr_write(struct ixgbe_softc *sc, uint32_t index, uint32_t itr)
991 {
992
993 /*
994 * Newer devices than 82598 have VF function, so this function is
995 * simple.
996 */
997 itr |= IXGBE_EITR_CNT_WDIS;
998
999 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(index), itr);
1000 }
1001
1002
1003 /************************************************************************
1004 * ixv_media_status - Media Ioctl callback
1005 *
1006 * Called whenever the user queries the status of
1007 * the interface using ifconfig.
1008 ************************************************************************/
1009 static void
ixv_media_status(struct ifnet * ifp,struct ifmediareq * ifmr)1010 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1011 {
1012 struct ixgbe_softc *sc = ifp->if_softc;
1013
1014 INIT_DEBUGOUT("ixv_media_status: begin");
1015 ixv_update_link_status(sc);
1016
1017 ifmr->ifm_status = IFM_AVALID;
1018 ifmr->ifm_active = IFM_ETHER;
1019
1020 if (sc->link_active != LINK_STATE_UP) {
1021 ifmr->ifm_active |= IFM_NONE;
1022 return;
1023 }
1024
1025 ifmr->ifm_status |= IFM_ACTIVE;
1026
1027 switch (sc->link_speed) {
1028 case IXGBE_LINK_SPEED_10GB_FULL:
1029 ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
1030 break;
1031 case IXGBE_LINK_SPEED_5GB_FULL:
1032 ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
1033 break;
1034 case IXGBE_LINK_SPEED_2_5GB_FULL:
1035 ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
1036 break;
1037 case IXGBE_LINK_SPEED_1GB_FULL:
1038 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1039 break;
1040 case IXGBE_LINK_SPEED_100_FULL:
1041 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1042 break;
1043 case IXGBE_LINK_SPEED_10_FULL:
1044 ifmr->ifm_active |= IFM_10_T | IFM_FDX;
1045 break;
1046 }
1047
1048 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active);
1049 } /* ixv_media_status */
1050
1051 /************************************************************************
1052 * ixv_media_change - Media Ioctl callback
1053 *
1054 * Called when the user changes speed/duplex using
1055 * media/mediopt option with ifconfig.
1056 ************************************************************************/
1057 static int
ixv_media_change(struct ifnet * ifp)1058 ixv_media_change(struct ifnet *ifp)
1059 {
1060 struct ixgbe_softc *sc = ifp->if_softc;
1061 struct ifmedia *ifm = &sc->media;
1062
1063 INIT_DEBUGOUT("ixv_media_change: begin");
1064
1065 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1066 return (EINVAL);
1067
1068 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1069 case IFM_AUTO:
1070 break;
1071 default:
1072 device_printf(sc->dev, "Only auto media type\n");
1073 return (EINVAL);
1074 }
1075
1076 return (0);
1077 } /* ixv_media_change */
1078
1079 static void
ixv_schedule_admin_tasklet(struct ixgbe_softc * sc)1080 ixv_schedule_admin_tasklet(struct ixgbe_softc *sc)
1081 {
1082 if (sc->schedule_wqs_ok) {
1083 if (atomic_cas_uint(&sc->admin_pending, 0, 1) == 0)
1084 workqueue_enqueue(sc->admin_wq,
1085 &sc->admin_wc, NULL);
1086 }
1087 }
1088
1089 /************************************************************************
1090 * ixv_negotiate_api
1091 *
1092 * Negotiate the Mailbox API with the PF;
1093 * start with the most featured API first.
1094 ************************************************************************/
1095 static int
ixv_negotiate_api(struct ixgbe_softc * sc)1096 ixv_negotiate_api(struct ixgbe_softc *sc)
1097 {
1098 struct ixgbe_hw *hw = &sc->hw;
1099 int mbx_api[] = { ixgbe_mbox_api_15,
1100 ixgbe_mbox_api_13,
1101 ixgbe_mbox_api_12,
1102 ixgbe_mbox_api_11,
1103 ixgbe_mbox_api_10,
1104 ixgbe_mbox_api_unknown };
1105 int i = 0;
1106
1107 while (mbx_api[i] != ixgbe_mbox_api_unknown) {
1108 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) {
1109 if (hw->api_version >= ixgbe_mbox_api_15)
1110 ixgbe_upgrade_mbx_params_vf(hw);
1111 return (0);
1112 }
1113 i++;
1114 }
1115
1116 return (EINVAL);
1117 } /* ixv_negotiate_api */
1118
1119
1120 /************************************************************************
1121 * ixv_set_rxfilter - Multicast Update
1122 *
1123 * Called whenever multicast address list is updated.
1124 ************************************************************************/
1125 static int
ixv_set_rxfilter(struct ixgbe_softc * sc)1126 ixv_set_rxfilter(struct ixgbe_softc *sc)
1127 {
1128 struct ixgbe_mc_addr *mta;
1129 struct ifnet *ifp = sc->ifp;
1130 struct ixgbe_hw *hw = &sc->hw;
1131 u8 *update_ptr;
1132 int mcnt = 0;
1133 struct ethercom *ec = &sc->osdep.ec;
1134 struct ether_multi *enm;
1135 struct ether_multistep step;
1136 bool overflow = false;
1137 int error, rc = 0;
1138
1139 KASSERT(mutex_owned(&sc->core_mtx));
1140 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin");
1141
1142 mta = sc->mta;
1143 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC);
1144
1145 /* 1: For PROMISC */
1146 if (ifp->if_flags & IFF_PROMISC) {
1147 error = hw->mac.ops.update_xcast_mode(hw,
1148 IXGBEVF_XCAST_MODE_PROMISC);
1149 if (error == IXGBE_ERR_NOT_TRUSTED) {
1150 device_printf(sc->dev,
1151 "this interface is not trusted\n");
1152 error = EPERM;
1153 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) {
1154 device_printf(sc->dev,
1155 "the PF doesn't support promisc mode\n");
1156 error = EOPNOTSUPP;
1157 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) {
1158 device_printf(sc->dev,
1159 "the PF may not in promisc mode\n");
1160 error = EINVAL;
1161 } else if (error) {
1162 device_printf(sc->dev,
1163 "failed to set promisc mode. error = %d\n",
1164 error);
1165 error = EIO;
1166 } else
1167 return 0;
1168 rc = error;
1169 }
1170
1171 /* 2: For ALLMULTI or normal */
1172 ETHER_LOCK(ec);
1173 ETHER_FIRST_MULTI(step, ec, enm);
1174 while (enm != NULL) {
1175 if ((mcnt >= IXGBE_MAX_VF_MC) ||
1176 (memcmp(enm->enm_addrlo, enm->enm_addrhi,
1177 ETHER_ADDR_LEN) != 0)) {
1178 overflow = true;
1179 break;
1180 }
1181 bcopy(enm->enm_addrlo,
1182 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
1183 mcnt++;
1184 ETHER_NEXT_MULTI(step, enm);
1185 }
1186 ETHER_UNLOCK(ec);
1187
1188 /* 3: For ALLMULTI */
1189 if (overflow) {
1190 error = hw->mac.ops.update_xcast_mode(hw,
1191 IXGBEVF_XCAST_MODE_ALLMULTI);
1192 if (error == IXGBE_ERR_NOT_TRUSTED) {
1193 device_printf(sc->dev,
1194 "this interface is not trusted\n");
1195 error = EPERM;
1196 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) {
1197 device_printf(sc->dev,
1198 "the PF doesn't support allmulti mode\n");
1199 error = EOPNOTSUPP;
1200 } else if (error) {
1201 device_printf(sc->dev,
1202 "number of Ethernet multicast addresses "
1203 "exceeds the limit (%d). error = %d\n",
1204 IXGBE_MAX_VF_MC, error);
1205 error = ENOSPC;
1206 } else {
1207 ETHER_LOCK(ec);
1208 ec->ec_flags |= ETHER_F_ALLMULTI;
1209 ETHER_UNLOCK(ec);
1210 return rc; /* Promisc might have failed */
1211 }
1212
1213 if (rc == 0)
1214 rc = error;
1215
1216 /* Continue to update the multicast table as many as we can */
1217 }
1218
1219 /* 4: For normal operation */
1220 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI);
1221 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) {
1222 /* Normal operation */
1223 ETHER_LOCK(ec);
1224 ec->ec_flags &= ~ETHER_F_ALLMULTI;
1225 ETHER_UNLOCK(ec);
1226 error = 0;
1227 } else if (error) {
1228 device_printf(sc->dev,
1229 "failed to set Ethernet multicast address "
1230 "operation to normal. error = %d\n", error);
1231 }
1232
1233 update_ptr = (u8 *)mta;
1234 error = sc->hw.mac.ops.update_mc_addr_list(&sc->hw,
1235 update_ptr, mcnt, ixv_mc_array_itr, TRUE);
1236 if (rc == 0)
1237 rc = error;
1238
1239 return rc;
1240 } /* ixv_set_rxfilter */
1241
1242 /************************************************************************
1243 * ixv_mc_array_itr
1244 *
1245 * An iterator function needed by the multicast shared code.
1246 * It feeds the shared code routine the addresses in the
1247 * array of ixv_set_rxfilter() one by one.
1248 ************************************************************************/
1249 static u8 *
ixv_mc_array_itr(struct ixgbe_hw * hw,u8 ** update_ptr,u32 * vmdq)1250 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1251 {
1252 struct ixgbe_mc_addr *mta;
1253
1254 mta = (struct ixgbe_mc_addr *)*update_ptr;
1255
1256 *vmdq = 0;
1257 *update_ptr = (u8*)(mta + 1);
1258
1259 return (mta->addr);
1260 } /* ixv_mc_array_itr */
1261
1262 /************************************************************************
1263 * ixv_local_timer - Timer routine
1264 *
1265 * Checks for link status, updates statistics,
1266 * and runs the watchdog check.
1267 ************************************************************************/
1268 static void
ixv_local_timer(void * arg)1269 ixv_local_timer(void *arg)
1270 {
1271 struct ixgbe_softc *sc = arg;
1272
1273 if (sc->schedule_wqs_ok) {
1274 if (atomic_cas_uint(&sc->timer_pending, 0, 1) == 0)
1275 workqueue_enqueue(sc->timer_wq,
1276 &sc->timer_wc, NULL);
1277 }
1278 }
1279
1280 static void
ixv_handle_timer(struct work * wk,void * context)1281 ixv_handle_timer(struct work *wk, void *context)
1282 {
1283 struct ixgbe_softc *sc = context;
1284 device_t dev = sc->dev;
1285 struct ix_queue *que = sc->queues;
1286 u64 queues = 0;
1287 u64 v0, v1, v2, v3, v4, v5, v6, v7;
1288 int hung = 0;
1289 int i;
1290
1291 IXGBE_CORE_LOCK(sc);
1292
1293 if (ixv_check_link(sc)) {
1294 ixv_init_locked(sc);
1295 IXGBE_CORE_UNLOCK(sc);
1296 return;
1297 }
1298
1299 /* Stats Update */
1300 ixv_update_stats(sc);
1301
1302 /* Update some event counters */
1303 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0;
1304 que = sc->queues;
1305 for (i = 0; i < sc->num_queues; i++, que++) {
1306 struct tx_ring *txr = que->txr;
1307
1308 v0 += txr->q_efbig_tx_dma_setup;
1309 v1 += txr->q_mbuf_defrag_failed;
1310 v2 += txr->q_efbig2_tx_dma_setup;
1311 v3 += txr->q_einval_tx_dma_setup;
1312 v4 += txr->q_other_tx_dma_setup;
1313 v5 += txr->q_eagain_tx_dma_setup;
1314 v6 += txr->q_enomem_tx_dma_setup;
1315 v7 += txr->q_tso_err;
1316 }
1317 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, v0);
1318 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, v1);
1319 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, v2);
1320 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, v3);
1321 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, v4);
1322 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, v5);
1323 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, v6);
1324 IXGBE_EVC_STORE(&sc->tso_err, v7);
1325
1326 /*
1327 * Check the TX queues status
1328 * - mark hung queues so we don't schedule on them
1329 * - watchdog only if all queues show hung
1330 */
1331 que = sc->queues;
1332 for (i = 0; i < sc->num_queues; i++, que++) {
1333 /* Keep track of queues with work for soft irq */
1334 if (que->txr->busy)
1335 queues |= ((u64)1 << que->me);
1336 /*
1337 * Each time txeof runs without cleaning, but there
1338 * are uncleaned descriptors it increments busy. If
1339 * we get to the MAX we declare it hung.
1340 */
1341 if (que->busy == IXGBE_QUEUE_HUNG) {
1342 ++hung;
1343 /* Mark the queue as inactive */
1344 sc->active_queues &= ~((u64)1 << que->me);
1345 continue;
1346 } else {
1347 /* Check if we've come back from hung */
1348 if ((sc->active_queues & ((u64)1 << que->me)) == 0)
1349 sc->active_queues |= ((u64)1 << que->me);
1350 }
1351 if (que->busy >= IXGBE_MAX_TX_BUSY) {
1352 device_printf(dev,
1353 "Warning queue %d appears to be hung!\n", i);
1354 que->txr->busy = IXGBE_QUEUE_HUNG;
1355 ++hung;
1356 }
1357 }
1358
1359 /* Only truly watchdog if all queues show hung */
1360 if (hung == sc->num_queues)
1361 goto watchdog;
1362 #if 0
1363 else if (queues != 0) { /* Force an IRQ on queues with work */
1364 ixv_rearm_queues(sc, queues);
1365 }
1366 #endif
1367
1368 atomic_store_relaxed(&sc->timer_pending, 0);
1369 IXGBE_CORE_UNLOCK(sc);
1370 callout_reset(&sc->timer, hz, ixv_local_timer, sc);
1371
1372 return;
1373
1374 watchdog:
1375 device_printf(sc->dev, "Watchdog timeout -- resetting\n");
1376 sc->ifp->if_flags &= ~IFF_RUNNING;
1377 IXGBE_EVC_ADD(&sc->watchdog_events, 1);
1378 ixv_init_locked(sc);
1379 IXGBE_CORE_UNLOCK(sc);
1380 } /* ixv_handle_timer */
1381
1382 /************************************************************************
1383 * ixv_update_link_status - Update OS on link state
1384 *
1385 * Note: Only updates the OS on the cached link state.
1386 * The real check of the hardware only happens with
1387 * a link interrupt.
1388 ************************************************************************/
1389 static void
ixv_update_link_status(struct ixgbe_softc * sc)1390 ixv_update_link_status(struct ixgbe_softc *sc)
1391 {
1392 struct ifnet *ifp = sc->ifp;
1393 device_t dev = sc->dev;
1394
1395 KASSERT(mutex_owned(&sc->core_mtx));
1396
1397 if (sc->link_up && sc->link_enabled) {
1398 if (sc->link_active != LINK_STATE_UP) {
1399 if (bootverbose) {
1400 const char *bpsmsg;
1401
1402 switch (sc->link_speed) {
1403 case IXGBE_LINK_SPEED_10GB_FULL:
1404 bpsmsg = "10 Gbps";
1405 break;
1406 case IXGBE_LINK_SPEED_5GB_FULL:
1407 bpsmsg = "5 Gbps";
1408 break;
1409 case IXGBE_LINK_SPEED_2_5GB_FULL:
1410 bpsmsg = "2.5 Gbps";
1411 break;
1412 case IXGBE_LINK_SPEED_1GB_FULL:
1413 bpsmsg = "1 Gbps";
1414 break;
1415 case IXGBE_LINK_SPEED_100_FULL:
1416 bpsmsg = "100 Mbps";
1417 break;
1418 case IXGBE_LINK_SPEED_10_FULL:
1419 bpsmsg = "10 Mbps";
1420 break;
1421 default:
1422 bpsmsg = "unknown speed";
1423 break;
1424 }
1425 device_printf(dev, "Link is up %s %s \n",
1426 bpsmsg, "Full Duplex");
1427 }
1428 sc->link_active = LINK_STATE_UP;
1429 if_link_state_change(ifp, LINK_STATE_UP);
1430 }
1431 } else {
1432 /*
1433 * Do it when link active changes to DOWN. i.e.
1434 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN
1435 * b) LINK_STATE_UP -> LINK_STATE_DOWN
1436 */
1437 if (sc->link_active != LINK_STATE_DOWN) {
1438 if (bootverbose)
1439 device_printf(dev, "Link is Down\n");
1440 if_link_state_change(ifp, LINK_STATE_DOWN);
1441 sc->link_active = LINK_STATE_DOWN;
1442 }
1443 }
1444 } /* ixv_update_link_status */
1445
1446
1447 /************************************************************************
1448 * ixv_stop - Stop the hardware
1449 *
1450 * Disables all traffic on the adapter by issuing a
1451 * global reset on the MAC and deallocates TX/RX buffers.
1452 ************************************************************************/
1453 static void
ixv_ifstop(struct ifnet * ifp,int disable)1454 ixv_ifstop(struct ifnet *ifp, int disable)
1455 {
1456 struct ixgbe_softc *sc = ifp->if_softc;
1457
1458 IXGBE_CORE_LOCK(sc);
1459 ixv_stop_locked(sc);
1460 IXGBE_CORE_UNLOCK(sc);
1461
1462 workqueue_wait(sc->admin_wq, &sc->admin_wc);
1463 atomic_store_relaxed(&sc->admin_pending, 0);
1464 workqueue_wait(sc->timer_wq, &sc->timer_wc);
1465 atomic_store_relaxed(&sc->timer_pending, 0);
1466 }
1467
1468 static void
ixv_stop_locked(void * arg)1469 ixv_stop_locked(void *arg)
1470 {
1471 struct ifnet *ifp;
1472 struct ixgbe_softc *sc = arg;
1473 struct ixgbe_hw *hw = &sc->hw;
1474
1475 ifp = sc->ifp;
1476
1477 KASSERT(mutex_owned(&sc->core_mtx));
1478
1479 INIT_DEBUGOUT("ixv_stop_locked: begin\n");
1480 ixv_disable_intr(sc);
1481
1482 /* Tell the stack that the interface is no longer active */
1483 ifp->if_flags &= ~IFF_RUNNING;
1484
1485 hw->mac.ops.reset_hw(hw);
1486 sc->hw.adapter_stopped = FALSE;
1487 hw->mac.ops.stop_adapter(hw);
1488 callout_stop(&sc->timer);
1489
1490 /* Don't schedule workqueues. */
1491 sc->schedule_wqs_ok = false;
1492
1493 /* reprogram the RAR[0] in case user changed it. */
1494 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
1495
1496 return;
1497 } /* ixv_stop_locked */
1498
1499
1500 /************************************************************************
1501 * ixv_allocate_pci_resources
1502 ************************************************************************/
1503 static int
ixv_allocate_pci_resources(struct ixgbe_softc * sc,const struct pci_attach_args * pa)1504 ixv_allocate_pci_resources(struct ixgbe_softc *sc,
1505 const struct pci_attach_args *pa)
1506 {
1507 pcireg_t memtype, csr;
1508 device_t dev = sc->dev;
1509 bus_addr_t addr;
1510 int flags;
1511
1512 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0));
1513 switch (memtype) {
1514 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT:
1515 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT:
1516 sc->osdep.mem_bus_space_tag = pa->pa_memt;
1517 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0),
1518 memtype, &addr, &sc->osdep.mem_size, &flags) != 0)
1519 goto map_err;
1520 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
1521 aprint_normal_dev(dev, "clearing prefetchable bit\n");
1522 flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
1523 }
1524 if (bus_space_map(sc->osdep.mem_bus_space_tag, addr,
1525 sc->osdep.mem_size, flags,
1526 &sc->osdep.mem_bus_space_handle) != 0) {
1527 map_err:
1528 sc->osdep.mem_size = 0;
1529 aprint_error_dev(dev, "unable to map BAR0\n");
1530 return ENXIO;
1531 }
1532 /*
1533 * Enable address decoding for memory range in case it's not
1534 * set.
1535 */
1536 csr = pci_conf_read(pa->pa_pc, pa->pa_tag,
1537 PCI_COMMAND_STATUS_REG);
1538 csr |= PCI_COMMAND_MEM_ENABLE;
1539 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
1540 csr);
1541 break;
1542 default:
1543 aprint_error_dev(dev, "unexpected type on BAR0\n");
1544 return ENXIO;
1545 }
1546
1547 /* Pick up the tuneable queues */
1548 sc->num_queues = ixv_num_queues;
1549
1550 return (0);
1551 } /* ixv_allocate_pci_resources */
1552
1553 static void
ixv_free_deferred_handlers(struct ixgbe_softc * sc)1554 ixv_free_deferred_handlers(struct ixgbe_softc *sc)
1555 {
1556 struct ix_queue *que = sc->queues;
1557 struct tx_ring *txr = sc->tx_rings;
1558 int i;
1559
1560 for (i = 0; i < sc->num_queues; i++, que++, txr++) {
1561 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) {
1562 if (txr->txr_si != NULL)
1563 softint_disestablish(txr->txr_si);
1564 }
1565 if (que->que_si != NULL)
1566 softint_disestablish(que->que_si);
1567 }
1568 if (sc->txr_wq != NULL)
1569 workqueue_destroy(sc->txr_wq);
1570 if (sc->txr_wq_enqueued != NULL)
1571 percpu_free(sc->txr_wq_enqueued, sizeof(u_int));
1572 if (sc->que_wq != NULL)
1573 workqueue_destroy(sc->que_wq);
1574
1575 /* Drain the Mailbox(link) queue */
1576 if (sc->admin_wq != NULL) {
1577 workqueue_destroy(sc->admin_wq);
1578 sc->admin_wq = NULL;
1579 }
1580 if (sc->timer_wq != NULL) {
1581 workqueue_destroy(sc->timer_wq);
1582 sc->timer_wq = NULL;
1583 }
1584 } /* ixv_free_deferred_handlers */
1585
1586 /************************************************************************
1587 * ixv_free_pci_resources
1588 ************************************************************************/
1589 static void
ixv_free_pci_resources(struct ixgbe_softc * sc)1590 ixv_free_pci_resources(struct ixgbe_softc *sc)
1591 {
1592 struct ix_queue *que = sc->queues;
1593 int rid;
1594
1595 /*
1596 * Release all msix queue resources:
1597 */
1598 for (int i = 0; i < sc->num_queues; i++, que++) {
1599 if (que->res != NULL)
1600 pci_intr_disestablish(sc->osdep.pc,
1601 sc->osdep.ihs[i]);
1602 }
1603
1604
1605 /* Clean the Mailbox interrupt last */
1606 rid = sc->vector;
1607
1608 if (sc->osdep.ihs[rid] != NULL) {
1609 pci_intr_disestablish(sc->osdep.pc,
1610 sc->osdep.ihs[rid]);
1611 sc->osdep.ihs[rid] = NULL;
1612 }
1613
1614 pci_intr_release(sc->osdep.pc, sc->osdep.intrs,
1615 sc->osdep.nintrs);
1616
1617 if (sc->osdep.mem_size != 0) {
1618 bus_space_unmap(sc->osdep.mem_bus_space_tag,
1619 sc->osdep.mem_bus_space_handle,
1620 sc->osdep.mem_size);
1621 }
1622
1623 return;
1624 } /* ixv_free_pci_resources */
1625
1626 /************************************************************************
1627 * ixv_setup_interface
1628 *
1629 * Setup networking device structure and register an interface.
1630 ************************************************************************/
1631 static int
ixv_setup_interface(device_t dev,struct ixgbe_softc * sc)1632 ixv_setup_interface(device_t dev, struct ixgbe_softc *sc)
1633 {
1634 struct ethercom *ec = &sc->osdep.ec;
1635 struct ifnet *ifp;
1636
1637 INIT_DEBUGOUT("ixv_setup_interface: begin");
1638
1639 ifp = sc->ifp = &ec->ec_if;
1640 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ);
1641 ifp->if_baudrate = IF_Gbps(10);
1642 ifp->if_init = ixv_init;
1643 ifp->if_stop = ixv_ifstop;
1644 ifp->if_softc = sc;
1645 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1646 ifp->if_extflags = IFEF_MPSAFE;
1647 ifp->if_ioctl = ixv_ioctl;
1648 if (sc->feat_en & IXGBE_FEATURE_LEGACY_TX) {
1649 #if 0
1650 ixv_start_locked = ixgbe_legacy_start_locked;
1651 #endif
1652 } else {
1653 ifp->if_transmit = ixgbe_mq_start;
1654 #if 0
1655 ixv_start_locked = ixgbe_mq_start_locked;
1656 #endif
1657 }
1658 ifp->if_start = ixgbe_legacy_start;
1659 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 2);
1660 IFQ_SET_READY(&ifp->if_snd);
1661
1662 if_initialize(ifp);
1663 sc->ipq = if_percpuq_create(&sc->osdep.ec.ec_if);
1664 /*
1665 * We use per TX queue softint, so if_deferred_start_init() isn't
1666 * used.
1667 */
1668
1669 sc->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR;
1670
1671 /*
1672 * Tell the upper layer(s) we support long frames.
1673 */
1674 ifp->if_hdrlen = sizeof(struct ether_vlan_header);
1675
1676 /* Set capability flags */
1677 ifp->if_capabilities |= IFCAP_HWCSUM
1678 | IFCAP_TSOv4
1679 | IFCAP_TSOv6;
1680 ifp->if_capenable = 0;
1681
1682 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER
1683 | ETHERCAP_VLAN_HWTAGGING
1684 | ETHERCAP_VLAN_HWCSUM
1685 | ETHERCAP_JUMBO_MTU
1686 | ETHERCAP_VLAN_MTU;
1687
1688 /* Enable the above capabilities by default */
1689 ec->ec_capenable = ec->ec_capabilities;
1690
1691 ether_ifattach(ifp, sc->hw.mac.addr);
1692 aprint_normal_dev(dev, "Ethernet address %s\n",
1693 ether_sprintf(sc->hw.mac.addr));
1694 ether_set_ifflags_cb(ec, ixv_ifflags_cb);
1695
1696 /* Don't enable LRO by default */
1697 #if 0
1698 /* NetBSD doesn't support LRO yet */
1699 ifp->if_capabilities |= IFCAP_LRO;
1700 #endif
1701
1702 /*
1703 * Specify the media types supported by this adapter and register
1704 * callbacks to update media and link information
1705 */
1706 ec->ec_ifmedia = &sc->media;
1707 ifmedia_init_with_lock(&sc->media, IFM_IMASK, ixv_media_change,
1708 ixv_media_status, &sc->core_mtx);
1709 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1710 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
1711
1712 if_register(ifp);
1713
1714 return 0;
1715 } /* ixv_setup_interface */
1716
1717
1718 /************************************************************************
1719 * ixv_initialize_transmit_units - Enable transmit unit.
1720 ************************************************************************/
1721 static void
ixv_initialize_transmit_units(struct ixgbe_softc * sc)1722 ixv_initialize_transmit_units(struct ixgbe_softc *sc)
1723 {
1724 struct tx_ring *txr = sc->tx_rings;
1725 struct ixgbe_hw *hw = &sc->hw;
1726 int i;
1727
1728 for (i = 0; i < sc->num_queues; i++, txr++) {
1729 u64 tdba = txr->txdma.dma_paddr;
1730 u32 txctrl, txdctl;
1731 int j = txr->me;
1732
1733 /* Set WTHRESH to 8, burst writeback */
1734 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1735 txdctl &= ~IXGBE_TXDCTL_WTHRESH_MASK;
1736 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT;
1737 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1738
1739 /* Set the HW Tx Head and Tail indices */
1740 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0);
1741 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0);
1742
1743 /* Set Tx Tail register */
1744 txr->tail = IXGBE_VFTDT(j);
1745
1746 txr->txr_no_space = false;
1747
1748 /* Set Ring parameters */
1749 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1750 (tdba & 0x00000000ffffffffULL));
1751 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1752 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j),
1753 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
1754 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1755 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
1756 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1757
1758 /* Now enable */
1759 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1760 txdctl |= IXGBE_TXDCTL_ENABLE;
1761 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1762 }
1763
1764 return;
1765 } /* ixv_initialize_transmit_units */
1766
1767
1768 /************************************************************************
1769 * ixv_initialize_rss_mapping
1770 ************************************************************************/
1771 static void
ixv_initialize_rss_mapping(struct ixgbe_softc * sc)1772 ixv_initialize_rss_mapping(struct ixgbe_softc *sc)
1773 {
1774 struct ixgbe_hw *hw = &sc->hw;
1775 u32 reta = 0, mrqc, rss_key[10];
1776 int queue_id;
1777 int i, j;
1778 u32 rss_hash_config;
1779
1780 /* force use default RSS key. */
1781 #ifdef __NetBSD__
1782 rss_getkey((uint8_t *) &rss_key);
1783 #else
1784 if (sc->feat_en & IXGBE_FEATURE_RSS) {
1785 /* Fetch the configured RSS key */
1786 rss_getkey((uint8_t *)&rss_key);
1787 } else {
1788 /* set up random bits */
1789 cprng_fast(&rss_key, sizeof(rss_key));
1790 }
1791 #endif
1792
1793 /* Now fill out hash function seeds */
1794 for (i = 0; i < 10; i++)
1795 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]);
1796
1797 /* Set up the redirection table */
1798 for (i = 0, j = 0; i < 64; i++, j++) {
1799 if (j == sc->num_queues)
1800 j = 0;
1801
1802 if (sc->feat_en & IXGBE_FEATURE_RSS) {
1803 /*
1804 * Fetch the RSS bucket id for the given indirection
1805 * entry. Cap it at the number of configured buckets
1806 * (which is num_queues.)
1807 */
1808 queue_id = rss_get_indirection_to_bucket(i);
1809 queue_id = queue_id % sc->num_queues;
1810 } else
1811 queue_id = j;
1812
1813 /*
1814 * The low 8 bits are for hash value (n+0);
1815 * The next 8 bits are for hash value (n+1), etc.
1816 */
1817 reta >>= 8;
1818 reta |= ((uint32_t)queue_id) << 24;
1819 if ((i & 3) == 3) {
1820 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta);
1821 reta = 0;
1822 }
1823 }
1824
1825 /* Perform hash on these packet types */
1826 if (sc->feat_en & IXGBE_FEATURE_RSS)
1827 rss_hash_config = rss_gethashconfig();
1828 else {
1829 /*
1830 * Disable UDP - IP fragments aren't currently being handled
1831 * and so we end up with a mix of 2-tuple and 4-tuple
1832 * traffic.
1833 */
1834 rss_hash_config = RSS_HASHTYPE_RSS_IPV4
1835 | RSS_HASHTYPE_RSS_TCP_IPV4
1836 | RSS_HASHTYPE_RSS_IPV6
1837 | RSS_HASHTYPE_RSS_TCP_IPV6;
1838 }
1839
1840 mrqc = IXGBE_MRQC_RSSEN;
1841 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
1842 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
1843 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
1844 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
1845 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
1846 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
1847 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
1848 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
1849 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
1850 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX "
1851 "defined, but not supported\n", __func__);
1852 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
1853 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX "
1854 "defined, but not supported\n", __func__);
1855 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
1856 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
1857 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
1858 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
1859 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
1860 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX "
1861 "defined, but not supported\n", __func__);
1862 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc);
1863 } /* ixv_initialize_rss_mapping */
1864
1865
1866 /************************************************************************
1867 * ixv_initialize_receive_units - Setup receive registers and features.
1868 ************************************************************************/
1869 static void
ixv_initialize_receive_units(struct ixgbe_softc * sc)1870 ixv_initialize_receive_units(struct ixgbe_softc *sc)
1871 {
1872 struct rx_ring *rxr = sc->rx_rings;
1873 struct ixgbe_hw *hw = &sc->hw;
1874 struct ifnet *ifp = sc->ifp;
1875 u32 bufsz, psrtype;
1876
1877 if (ifp->if_mtu > ETHERMTU)
1878 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1879 else
1880 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1881
1882 psrtype = IXGBE_PSRTYPE_TCPHDR
1883 | IXGBE_PSRTYPE_UDPHDR
1884 | IXGBE_PSRTYPE_IPV4HDR
1885 | IXGBE_PSRTYPE_IPV6HDR
1886 | IXGBE_PSRTYPE_L2HDR;
1887
1888 if (sc->num_queues > 1)
1889 psrtype |= 1 << 29;
1890
1891 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1892
1893 /* Tell PF our max_frame size */
1894 if (ixgbevf_rlpml_set_vf(hw, sc->max_frame_size) != 0) {
1895 device_printf(sc->dev, "There is a problem with the PF "
1896 "setup. It is likely the receive unit for this VF will "
1897 "not function correctly.\n");
1898 }
1899
1900 for (int i = 0; i < sc->num_queues; i++, rxr++) {
1901 u64 rdba = rxr->rxdma.dma_paddr;
1902 u32 reg, rxdctl;
1903 int j = rxr->me;
1904
1905 /* Disable the queue */
1906 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1907 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1908 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1909 for (int k = 0; k < 10; k++) {
1910 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
1911 IXGBE_RXDCTL_ENABLE)
1912 msec_delay(1);
1913 else
1914 break;
1915 }
1916 IXGBE_WRITE_BARRIER(hw);
1917 /* Setup the Base and Length of the Rx Descriptor Ring */
1918 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1919 (rdba & 0x00000000ffffffffULL));
1920 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1921 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j),
1922 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
1923
1924 /* Reset the ring indices */
1925 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0);
1926 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0);
1927
1928 /* Set up the SRRCTL register */
1929 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j));
1930 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
1931 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
1932 reg |= bufsz;
1933 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1934 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg);
1935
1936 /* Capture Rx Tail index */
1937 rxr->tail = IXGBE_VFRDT(rxr->me);
1938
1939 /* Do the queue enabling last */
1940 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1941 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1942 for (int k = 0; k < 10; k++) {
1943 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) &
1944 IXGBE_RXDCTL_ENABLE)
1945 break;
1946 msec_delay(1);
1947 }
1948 IXGBE_WRITE_BARRIER(hw);
1949
1950 /* Set the Tail Pointer */
1951 #ifdef DEV_NETMAP
1952 /*
1953 * In netmap mode, we must preserve the buffers made
1954 * available to userspace before the if_init()
1955 * (this is true by default on the TX side, because
1956 * init makes all buffers available to userspace).
1957 *
1958 * netmap_reset() and the device specific routines
1959 * (e.g. ixgbe_setup_receive_rings()) map these
1960 * buffers at the end of the NIC ring, so here we
1961 * must set the RDT (tail) register to make sure
1962 * they are not overwritten.
1963 *
1964 * In this driver the NIC ring starts at RDH = 0,
1965 * RDT points to the last slot available for reception (?),
1966 * so RDT = num_rx_desc - 1 means the whole ring is available.
1967 */
1968 if ((sc->feat_en & IXGBE_FEATURE_NETMAP) &&
1969 (ifp->if_capenable & IFCAP_NETMAP)) {
1970 struct netmap_adapter *na = NA(sc->ifp);
1971 struct netmap_kring *kring = na->rx_rings[i];
1972 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
1973
1974 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t);
1975 } else
1976 #endif /* DEV_NETMAP */
1977 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me),
1978 sc->num_rx_desc - 1);
1979 }
1980
1981 if (sc->hw.mac.type >= ixgbe_mac_X550_vf)
1982 ixv_initialize_rss_mapping(sc);
1983 } /* ixv_initialize_receive_units */
1984
1985 /************************************************************************
1986 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function
1987 *
1988 * Retrieves the TDH value from the hardware
1989 ************************************************************************/
1990 static int
ixv_sysctl_tdh_handler(SYSCTLFN_ARGS)1991 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS)
1992 {
1993 struct sysctlnode node = *rnode;
1994 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data;
1995 uint32_t val;
1996
1997 if (!txr)
1998 return (0);
1999
2000 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDH(txr->me));
2001 node.sysctl_data = &val;
2002 return sysctl_lookup(SYSCTLFN_CALL(&node));
2003 } /* ixv_sysctl_tdh_handler */
2004
2005 /************************************************************************
2006 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function
2007 *
2008 * Retrieves the TDT value from the hardware
2009 ************************************************************************/
2010 static int
ixv_sysctl_tdt_handler(SYSCTLFN_ARGS)2011 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS)
2012 {
2013 struct sysctlnode node = *rnode;
2014 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data;
2015 uint32_t val;
2016
2017 if (!txr)
2018 return (0);
2019
2020 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDT(txr->me));
2021 node.sysctl_data = &val;
2022 return sysctl_lookup(SYSCTLFN_CALL(&node));
2023 } /* ixv_sysctl_tdt_handler */
2024
2025 /************************************************************************
2026 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check
2027 * handler function
2028 *
2029 * Retrieves the next_to_check value
2030 ************************************************************************/
2031 static int
ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS)2032 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS)
2033 {
2034 struct sysctlnode node = *rnode;
2035 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
2036 uint32_t val;
2037
2038 if (!rxr)
2039 return (0);
2040
2041 val = rxr->next_to_check;
2042 node.sysctl_data = &val;
2043 return sysctl_lookup(SYSCTLFN_CALL(&node));
2044 } /* ixv_sysctl_next_to_check_handler */
2045
2046 /************************************************************************
2047 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh
2048 * handler function
2049 *
2050 * Retrieves the next_to_refresh value
2051 ************************************************************************/
2052 static int
ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS)2053 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS)
2054 {
2055 struct sysctlnode node = *rnode;
2056 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
2057 struct ixgbe_softc *sc;
2058 uint32_t val;
2059
2060 if (!rxr)
2061 return (0);
2062
2063 sc = rxr->sc;
2064 if (ixgbe_fw_recovery_mode_swflag(sc))
2065 return (EPERM);
2066
2067 val = rxr->next_to_refresh;
2068 node.sysctl_data = &val;
2069 return sysctl_lookup(SYSCTLFN_CALL(&node));
2070 } /* ixv_sysctl_next_to_refresh_handler */
2071
2072 /************************************************************************
2073 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function
2074 *
2075 * Retrieves the RDH value from the hardware
2076 ************************************************************************/
2077 static int
ixv_sysctl_rdh_handler(SYSCTLFN_ARGS)2078 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS)
2079 {
2080 struct sysctlnode node = *rnode;
2081 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
2082 uint32_t val;
2083
2084 if (!rxr)
2085 return (0);
2086
2087 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDH(rxr->me));
2088 node.sysctl_data = &val;
2089 return sysctl_lookup(SYSCTLFN_CALL(&node));
2090 } /* ixv_sysctl_rdh_handler */
2091
2092 /************************************************************************
2093 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function
2094 *
2095 * Retrieves the RDT value from the hardware
2096 ************************************************************************/
2097 static int
ixv_sysctl_rdt_handler(SYSCTLFN_ARGS)2098 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS)
2099 {
2100 struct sysctlnode node = *rnode;
2101 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
2102 uint32_t val;
2103
2104 if (!rxr)
2105 return (0);
2106
2107 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDT(rxr->me));
2108 node.sysctl_data = &val;
2109 return sysctl_lookup(SYSCTLFN_CALL(&node));
2110 } /* ixv_sysctl_rdt_handler */
2111
2112 static void
ixv_setup_vlan_tagging(struct ixgbe_softc * sc)2113 ixv_setup_vlan_tagging(struct ixgbe_softc *sc)
2114 {
2115 struct ethercom *ec = &sc->osdep.ec;
2116 struct ixgbe_hw *hw = &sc->hw;
2117 struct rx_ring *rxr;
2118 u32 ctrl;
2119 int i;
2120 bool hwtagging;
2121
2122 /* Enable HW tagging only if any vlan is attached */
2123 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING)
2124 && VLAN_ATTACHED(ec);
2125
2126 /* Enable the queues */
2127 for (i = 0; i < sc->num_queues; i++) {
2128 rxr = &sc->rx_rings[i];
2129 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me));
2130 if (hwtagging)
2131 ctrl |= IXGBE_RXDCTL_VME;
2132 else
2133 ctrl &= ~IXGBE_RXDCTL_VME;
2134 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl);
2135 /*
2136 * Let Rx path know that it needs to store VLAN tag
2137 * as part of extra mbuf info.
2138 */
2139 rxr->vtag_strip = hwtagging ? TRUE : FALSE;
2140 }
2141 } /* ixv_setup_vlan_tagging */
2142
2143 /************************************************************************
2144 * ixv_setup_vlan_support
2145 ************************************************************************/
2146 static int
ixv_setup_vlan_support(struct ixgbe_softc * sc)2147 ixv_setup_vlan_support(struct ixgbe_softc *sc)
2148 {
2149 struct ethercom *ec = &sc->osdep.ec;
2150 struct ixgbe_hw *hw = &sc->hw;
2151 u32 vid, vfta, retry;
2152 struct vlanid_list *vlanidp;
2153 int rv, error = 0;
2154
2155 /*
2156 * This function is called from both if_init and ifflags_cb()
2157 * on NetBSD.
2158 */
2159
2160 /*
2161 * Part 1:
2162 * Setup VLAN HW tagging
2163 */
2164 ixv_setup_vlan_tagging(sc);
2165
2166 if (!VLAN_ATTACHED(ec))
2167 return 0;
2168
2169 /*
2170 * Part 2:
2171 * Setup VLAN HW filter
2172 */
2173 /* Cleanup shadow_vfta */
2174 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
2175 sc->shadow_vfta[i] = 0;
2176 /* Generate shadow_vfta from ec_vids */
2177 ETHER_LOCK(ec);
2178 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
2179 uint32_t idx;
2180
2181 idx = vlanidp->vid / 32;
2182 KASSERT(idx < IXGBE_VFTA_SIZE);
2183 sc->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32);
2184 }
2185 ETHER_UNLOCK(ec);
2186
2187 /*
2188 * A soft reset zero's out the VFTA, so
2189 * we need to repopulate it now.
2190 */
2191 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) {
2192 if (sc->shadow_vfta[i] == 0)
2193 continue;
2194 vfta = sc->shadow_vfta[i];
2195 /*
2196 * Reconstruct the vlan id's
2197 * based on the bits set in each
2198 * of the array ints.
2199 */
2200 for (int j = 0; j < 32; j++) {
2201 retry = 0;
2202 if ((vfta & ((u32)1 << j)) == 0)
2203 continue;
2204 vid = (i * 32) + j;
2205
2206 /* Call the shared code mailbox routine */
2207 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE,
2208 FALSE)) != 0) {
2209 if (++retry > 5) {
2210 device_printf(sc->dev,
2211 "%s: max retry exceeded\n",
2212 __func__);
2213 break;
2214 }
2215 }
2216 if (rv != 0) {
2217 device_printf(sc->dev,
2218 "failed to set vlan %d\n", vid);
2219 error = EACCES;
2220 }
2221 }
2222 }
2223 return error;
2224 } /* ixv_setup_vlan_support */
2225
2226 static int
ixv_vlan_cb(struct ethercom * ec,uint16_t vid,bool set)2227 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set)
2228 {
2229 struct ifnet *ifp = &ec->ec_if;
2230 struct ixgbe_softc *sc = ifp->if_softc;
2231 int rv;
2232
2233 if (set)
2234 rv = ixv_register_vlan(sc, vid);
2235 else
2236 rv = ixv_unregister_vlan(sc, vid);
2237
2238 if (rv != 0)
2239 return rv;
2240
2241 /*
2242 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0
2243 * or 0 to 1.
2244 */
2245 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0)))
2246 ixv_setup_vlan_tagging(sc);
2247
2248 return rv;
2249 }
2250
2251 /************************************************************************
2252 * ixv_register_vlan
2253 *
2254 * Run via a vlan config EVENT, it enables us to use the
2255 * HW Filter table since we can get the vlan id. This just
2256 * creates the entry in the soft version of the VFTA, init
2257 * will repopulate the real table.
2258 ************************************************************************/
2259 static int
ixv_register_vlan(struct ixgbe_softc * sc,u16 vtag)2260 ixv_register_vlan(struct ixgbe_softc *sc, u16 vtag)
2261 {
2262 struct ixgbe_hw *hw = &sc->hw;
2263 u16 index, bit;
2264 int error;
2265
2266 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
2267 return EINVAL;
2268 IXGBE_CORE_LOCK(sc);
2269 index = (vtag >> 5) & 0x7F;
2270 bit = vtag & 0x1F;
2271 sc->shadow_vfta[index] |= ((u32)1 << bit);
2272 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false);
2273 IXGBE_CORE_UNLOCK(sc);
2274
2275 if (error != 0) {
2276 device_printf(sc->dev, "failed to register vlan %hu\n", vtag);
2277 error = EACCES;
2278 }
2279 return error;
2280 } /* ixv_register_vlan */
2281
2282 /************************************************************************
2283 * ixv_unregister_vlan
2284 *
2285 * Run via a vlan unconfig EVENT, remove our entry
2286 * in the soft vfta.
2287 ************************************************************************/
2288 static int
ixv_unregister_vlan(struct ixgbe_softc * sc,u16 vtag)2289 ixv_unregister_vlan(struct ixgbe_softc *sc, u16 vtag)
2290 {
2291 struct ixgbe_hw *hw = &sc->hw;
2292 u16 index, bit;
2293 int error;
2294
2295 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
2296 return EINVAL;
2297
2298 IXGBE_CORE_LOCK(sc);
2299 index = (vtag >> 5) & 0x7F;
2300 bit = vtag & 0x1F;
2301 sc->shadow_vfta[index] &= ~((u32)1 << bit);
2302 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false);
2303 IXGBE_CORE_UNLOCK(sc);
2304
2305 if (error != 0) {
2306 device_printf(sc->dev, "failed to unregister vlan %hu\n",
2307 vtag);
2308 error = EIO;
2309 }
2310 return error;
2311 } /* ixv_unregister_vlan */
2312
2313 /************************************************************************
2314 * ixv_enable_intr
2315 ************************************************************************/
2316 static void
ixv_enable_intr(struct ixgbe_softc * sc)2317 ixv_enable_intr(struct ixgbe_softc *sc)
2318 {
2319 struct ixgbe_hw *hw = &sc->hw;
2320 struct ix_queue *que = sc->queues;
2321 u32 mask;
2322 int i;
2323
2324 /* For VTEIAC */
2325 mask = (1 << sc->vector);
2326 for (i = 0; i < sc->num_queues; i++, que++)
2327 mask |= (1 << que->msix);
2328 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask);
2329
2330 /* For VTEIMS */
2331 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector));
2332 que = sc->queues;
2333 for (i = 0; i < sc->num_queues; i++, que++)
2334 ixv_enable_queue(sc, que->msix);
2335
2336 IXGBE_WRITE_FLUSH(hw);
2337 } /* ixv_enable_intr */
2338
2339 /************************************************************************
2340 * ixv_disable_intr
2341 ************************************************************************/
2342 static void
ixv_disable_intr(struct ixgbe_softc * sc)2343 ixv_disable_intr(struct ixgbe_softc *sc)
2344 {
2345 struct ix_queue *que = sc->queues;
2346
2347 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0);
2348
2349 /* disable interrupts other than queues */
2350 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, sc->vector);
2351
2352 for (int i = 0; i < sc->num_queues; i++, que++)
2353 ixv_disable_queue(sc, que->msix);
2354
2355 IXGBE_WRITE_FLUSH(&sc->hw);
2356 } /* ixv_disable_intr */
2357
2358 /************************************************************************
2359 * ixv_set_ivar
2360 *
2361 * Setup the correct IVAR register for a particular MSI-X interrupt
2362 * - entry is the register array entry
2363 * - vector is the MSI-X vector for this queue
2364 * - type is RX/TX/MISC
2365 ************************************************************************/
2366 static void
ixv_set_ivar(struct ixgbe_softc * sc,u8 entry,u8 vector,s8 type)2367 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type)
2368 {
2369 struct ixgbe_hw *hw = &sc->hw;
2370 u32 ivar, index;
2371
2372 vector |= IXGBE_IVAR_ALLOC_VAL;
2373
2374 if (type == -1) { /* MISC IVAR */
2375 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
2376 ivar &= ~0xFF;
2377 ivar |= vector;
2378 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
2379 } else { /* RX/TX IVARS */
2380 index = (16 * (entry & 1)) + (8 * type);
2381 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1));
2382 ivar &= ~(0xffUL << index);
2383 ivar |= ((u32)vector << index);
2384 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar);
2385 }
2386 } /* ixv_set_ivar */
2387
2388 /************************************************************************
2389 * ixv_configure_ivars
2390 ************************************************************************/
2391 static void
ixv_configure_ivars(struct ixgbe_softc * sc)2392 ixv_configure_ivars(struct ixgbe_softc *sc)
2393 {
2394 struct ix_queue *que = sc->queues;
2395
2396 /* XXX We should sync EITR value calculation with ixgbe.c? */
2397
2398 for (int i = 0; i < sc->num_queues; i++, que++) {
2399 /* First the RX queue entry */
2400 ixv_set_ivar(sc, i, que->msix, 0);
2401 /* ... and the TX */
2402 ixv_set_ivar(sc, i, que->msix, 1);
2403 /* Set an initial value in EITR */
2404 ixv_eitr_write(sc, que->msix, IXGBE_EITR_DEFAULT);
2405 }
2406
2407 /* For the mailbox interrupt */
2408 ixv_set_ivar(sc, 1, sc->vector, -1);
2409 } /* ixv_configure_ivars */
2410
2411
2412 /************************************************************************
2413 * ixv_init_stats
2414 *
2415 * The VF stats registers never have a truly virgin
2416 * starting point, so this routine save initial vaules to
2417 * last_<REGNAME>.
2418 ************************************************************************/
2419 static void
ixv_init_stats(struct ixgbe_softc * sc)2420 ixv_init_stats(struct ixgbe_softc *sc)
2421 {
2422 struct ixgbe_hw *hw = &sc->hw;
2423
2424 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
2425 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
2426 sc->stats.vf.last_vfgorc |=
2427 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
2428
2429 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
2430 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
2431 sc->stats.vf.last_vfgotc |=
2432 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
2433
2434 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
2435 } /* ixv_init_stats */
2436
2437 #define UPDATE_STAT_32(reg, last, count) \
2438 { \
2439 u32 current = IXGBE_READ_REG(hw, (reg)); \
2440 IXGBE_EVC_ADD(&count, current - (last)); \
2441 (last) = current; \
2442 }
2443
2444 #define UPDATE_STAT_36(lsb, msb, last, count) \
2445 { \
2446 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \
2447 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \
2448 u64 current = ((cur_msb << 32) | cur_lsb); \
2449 if (current < (last)) \
2450 IXGBE_EVC_ADD(&count, current + __BIT(36) - (last)); \
2451 else \
2452 IXGBE_EVC_ADD(&count, current - (last)); \
2453 (last) = current; \
2454 }
2455
2456 /************************************************************************
2457 * ixv_update_stats - Update the board statistics counters.
2458 ************************************************************************/
2459 void
ixv_update_stats(struct ixgbe_softc * sc)2460 ixv_update_stats(struct ixgbe_softc *sc)
2461 {
2462 struct ixgbe_hw *hw = &sc->hw;
2463 struct ixgbevf_hw_stats *stats = &sc->stats.vf;
2464
2465 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc);
2466 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc);
2467 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc,
2468 stats->vfgorc);
2469 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc,
2470 stats->vfgotc);
2471 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc);
2472
2473 /* VF doesn't count errors by hardware */
2474
2475 } /* ixv_update_stats */
2476
2477 /************************************************************************
2478 * ixv_sysctl_interrupt_rate_handler
2479 ************************************************************************/
2480 static int
ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS)2481 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS)
2482 {
2483 struct sysctlnode node = *rnode;
2484 struct ix_queue *que = (struct ix_queue *)node.sysctl_data;
2485 struct ixgbe_softc *sc = que->sc;
2486 uint32_t reg, usec, rate;
2487 int error;
2488
2489 if (que == NULL)
2490 return 0;
2491 reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_VTEITR(que->msix));
2492 usec = ((reg & 0x0FF8) >> 3);
2493 if (usec > 0)
2494 rate = 500000 / usec;
2495 else
2496 rate = 0;
2497 node.sysctl_data = &rate;
2498 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2499 if (error || newp == NULL)
2500 return error;
2501 reg &= ~0xfff; /* default, no limitation */
2502 if (rate > 0 && rate < 500000) {
2503 if (rate < 1000)
2504 rate = 1000;
2505 reg |= ((4000000 / rate) & 0xff8);
2506 /*
2507 * When RSC is used, ITR interval must be larger than
2508 * RSC_DELAY. Currently, we use 2us for RSC_DELAY.
2509 * The minimum value is always greater than 2us on 100M
2510 * (and 10M?(not documented)), but it's not on 1G and higher.
2511 */
2512 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL)
2513 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) {
2514 if ((sc->num_queues > 1)
2515 && (reg < IXGBE_MIN_RSC_EITR_10G1G))
2516 return EINVAL;
2517 }
2518 sc->max_interrupt_rate = rate;
2519 } else
2520 sc->max_interrupt_rate = 0;
2521 ixv_eitr_write(sc, que->msix, reg);
2522
2523 return (0);
2524 } /* ixv_sysctl_interrupt_rate_handler */
2525
2526 const struct sysctlnode *
ixv_sysctl_instance(struct ixgbe_softc * sc)2527 ixv_sysctl_instance(struct ixgbe_softc *sc)
2528 {
2529 const char *dvname;
2530 struct sysctllog **log;
2531 int rc;
2532 const struct sysctlnode *rnode;
2533
2534 log = &sc->sysctllog;
2535 dvname = device_xname(sc->dev);
2536
2537 if ((rc = sysctl_createv(log, 0, NULL, &rnode,
2538 0, CTLTYPE_NODE, dvname,
2539 SYSCTL_DESCR("ixv information and settings"),
2540 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0)
2541 goto err;
2542
2543 return rnode;
2544 err:
2545 device_printf(sc->dev,
2546 "%s: sysctl_createv failed, rc = %d\n", __func__, rc);
2547 return NULL;
2548 }
2549
2550 static void
ixv_add_device_sysctls(struct ixgbe_softc * sc)2551 ixv_add_device_sysctls(struct ixgbe_softc *sc)
2552 {
2553 struct sysctllog **log;
2554 const struct sysctlnode *rnode, *cnode;
2555 device_t dev;
2556
2557 dev = sc->dev;
2558 log = &sc->sysctllog;
2559
2560 if ((rnode = ixv_sysctl_instance(sc)) == NULL) {
2561 aprint_error_dev(dev, "could not create sysctl root\n");
2562 return;
2563 }
2564
2565 if (sysctl_createv(log, 0, &rnode, &cnode,
2566 CTLFLAG_READWRITE, CTLTYPE_INT, "debug",
2567 SYSCTL_DESCR("Debug Info"),
2568 ixv_sysctl_debug, 0, (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0)
2569 aprint_error_dev(dev, "could not create sysctl\n");
2570
2571 if (sysctl_createv(log, 0, &rnode, &cnode,
2572 CTLFLAG_READWRITE, CTLTYPE_INT,
2573 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"),
2574 ixv_sysctl_rx_copy_len, 0,
2575 (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0)
2576 aprint_error_dev(dev, "could not create sysctl\n");
2577
2578 if (sysctl_createv(log, 0, &rnode, &cnode,
2579 CTLFLAG_READONLY, CTLTYPE_INT,
2580 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"),
2581 NULL, 0, &sc->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0)
2582 aprint_error_dev(dev, "could not create sysctl\n");
2583
2584 if (sysctl_createv(log, 0, &rnode, &cnode,
2585 CTLFLAG_READONLY, CTLTYPE_INT,
2586 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"),
2587 NULL, 0, &sc->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0)
2588 aprint_error_dev(dev, "could not create sysctl\n");
2589
2590 if (sysctl_createv(log, 0, &rnode, &cnode,
2591 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit",
2592 SYSCTL_DESCR("max number of RX packets to process"),
2593 ixv_sysctl_rx_process_limit, 0, (void *)sc, 0, CTL_CREATE,
2594 CTL_EOL) != 0)
2595 aprint_error_dev(dev, "could not create sysctl\n");
2596
2597 if (sysctl_createv(log, 0, &rnode, &cnode,
2598 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit",
2599 SYSCTL_DESCR("max number of TX packets to process"),
2600 ixv_sysctl_tx_process_limit, 0, (void *)sc, 0, CTL_CREATE,
2601 CTL_EOL) != 0)
2602 aprint_error_dev(dev, "could not create sysctl\n");
2603
2604 if (sysctl_createv(log, 0, &rnode, &cnode,
2605 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim",
2606 SYSCTL_DESCR("Interrupt Moderation"),
2607 NULL, 0, &sc->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0)
2608 aprint_error_dev(dev, "could not create sysctl\n");
2609
2610 if (sysctl_createv(log, 0, &rnode, &cnode,
2611 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
2612 SYSCTL_DESCR("Use workqueue for packet processing"),
2613 NULL, 0, &sc->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL)
2614 != 0)
2615 aprint_error_dev(dev, "could not create sysctl\n");
2616 }
2617
2618 /************************************************************************
2619 * ixv_add_stats_sysctls - Add statistic sysctls for the VF.
2620 ************************************************************************/
2621 static void
ixv_add_stats_sysctls(struct ixgbe_softc * sc)2622 ixv_add_stats_sysctls(struct ixgbe_softc *sc)
2623 {
2624 device_t dev = sc->dev;
2625 struct tx_ring *txr = sc->tx_rings;
2626 struct rx_ring *rxr = sc->rx_rings;
2627 struct ixgbevf_hw_stats *stats = &sc->stats.vf;
2628 struct ixgbe_hw *hw = &sc->hw;
2629 const struct sysctlnode *rnode, *cnode;
2630 struct sysctllog **log = &sc->sysctllog;
2631 const char *xname = device_xname(dev);
2632
2633 /* Driver Statistics */
2634 evcnt_attach_dynamic(&sc->efbig_tx_dma_setup, EVCNT_TYPE_MISC,
2635 NULL, xname, "Driver tx dma soft fail EFBIG");
2636 evcnt_attach_dynamic(&sc->mbuf_defrag_failed, EVCNT_TYPE_MISC,
2637 NULL, xname, "m_defrag() failed");
2638 evcnt_attach_dynamic(&sc->efbig2_tx_dma_setup, EVCNT_TYPE_MISC,
2639 NULL, xname, "Driver tx dma hard fail EFBIG");
2640 evcnt_attach_dynamic(&sc->einval_tx_dma_setup, EVCNT_TYPE_MISC,
2641 NULL, xname, "Driver tx dma hard fail EINVAL");
2642 evcnt_attach_dynamic(&sc->other_tx_dma_setup, EVCNT_TYPE_MISC,
2643 NULL, xname, "Driver tx dma hard fail other");
2644 evcnt_attach_dynamic(&sc->eagain_tx_dma_setup, EVCNT_TYPE_MISC,
2645 NULL, xname, "Driver tx dma soft fail EAGAIN");
2646 evcnt_attach_dynamic(&sc->enomem_tx_dma_setup, EVCNT_TYPE_MISC,
2647 NULL, xname, "Driver tx dma soft fail ENOMEM");
2648 evcnt_attach_dynamic(&sc->watchdog_events, EVCNT_TYPE_MISC,
2649 NULL, xname, "Watchdog timeouts");
2650 evcnt_attach_dynamic(&sc->tso_err, EVCNT_TYPE_MISC,
2651 NULL, xname, "TSO errors");
2652 evcnt_attach_dynamic(&sc->admin_irqev, EVCNT_TYPE_INTR,
2653 NULL, xname, "Admin MSI-X IRQ Handled");
2654 evcnt_attach_dynamic(&sc->link_workev, EVCNT_TYPE_INTR,
2655 NULL, xname, "Admin event");
2656
2657 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) {
2658 #ifdef LRO
2659 struct lro_ctrl *lro = &rxr->lro;
2660 #endif
2661
2662 snprintf(sc->queues[i].evnamebuf,
2663 sizeof(sc->queues[i].evnamebuf), "%s q%d", xname, i);
2664 snprintf(sc->queues[i].namebuf,
2665 sizeof(sc->queues[i].namebuf), "q%d", i);
2666
2667 if ((rnode = ixv_sysctl_instance(sc)) == NULL) {
2668 aprint_error_dev(dev,
2669 "could not create sysctl root\n");
2670 break;
2671 }
2672
2673 if (sysctl_createv(log, 0, &rnode, &rnode,
2674 0, CTLTYPE_NODE,
2675 sc->queues[i].namebuf, SYSCTL_DESCR("Queue Name"),
2676 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0)
2677 break;
2678
2679 if (sysctl_createv(log, 0, &rnode, &cnode,
2680 CTLFLAG_READWRITE, CTLTYPE_INT,
2681 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"),
2682 ixv_sysctl_interrupt_rate_handler, 0,
2683 (void *)&sc->queues[i], 0, CTL_CREATE, CTL_EOL) != 0)
2684 break;
2685
2686 if (sysctl_createv(log, 0, &rnode, &cnode,
2687 CTLFLAG_READONLY, CTLTYPE_INT,
2688 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"),
2689 ixv_sysctl_tdh_handler, 0, (void *)txr,
2690 0, CTL_CREATE, CTL_EOL) != 0)
2691 break;
2692
2693 if (sysctl_createv(log, 0, &rnode, &cnode,
2694 CTLFLAG_READONLY, CTLTYPE_INT,
2695 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"),
2696 ixv_sysctl_tdt_handler, 0, (void *)txr,
2697 0, CTL_CREATE, CTL_EOL) != 0)
2698 break;
2699
2700 if (sysctl_createv(log, 0, &rnode, &cnode,
2701 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck",
2702 SYSCTL_DESCR("Receive Descriptor next to check"),
2703 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0,
2704 CTL_CREATE, CTL_EOL) != 0)
2705 break;
2706
2707 if (sysctl_createv(log, 0, &rnode, &cnode,
2708 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf",
2709 SYSCTL_DESCR("Receive Descriptor next to refresh"),
2710 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0,
2711 CTL_CREATE, CTL_EOL) != 0)
2712 break;
2713
2714 if (sysctl_createv(log, 0, &rnode, &cnode,
2715 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head",
2716 SYSCTL_DESCR("Receive Descriptor Head"),
2717 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0,
2718 CTL_CREATE, CTL_EOL) != 0)
2719 break;
2720
2721 if (sysctl_createv(log, 0, &rnode, &cnode,
2722 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail",
2723 SYSCTL_DESCR("Receive Descriptor Tail"),
2724 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0,
2725 CTL_CREATE, CTL_EOL) != 0)
2726 break;
2727
2728 evcnt_attach_dynamic(&sc->queues[i].irqs, EVCNT_TYPE_INTR,
2729 NULL, sc->queues[i].evnamebuf, "IRQs on queue");
2730 evcnt_attach_dynamic(&sc->queues[i].handleq,
2731 EVCNT_TYPE_MISC, NULL, sc->queues[i].evnamebuf,
2732 "Handled queue in softint");
2733 evcnt_attach_dynamic(&sc->queues[i].req, EVCNT_TYPE_MISC,
2734 NULL, sc->queues[i].evnamebuf, "Requeued in softint");
2735 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC,
2736 NULL, sc->queues[i].evnamebuf,
2737 "Queue Packets Transmitted");
2738 #ifndef IXGBE_LEGACY_TX
2739 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC,
2740 NULL, sc->queues[i].evnamebuf,
2741 "Packets dropped in pcq");
2742 #endif
2743 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC,
2744 NULL, sc->queues[i].evnamebuf,
2745 "TX Queue No Descriptor Available");
2746 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC,
2747 NULL, sc->queues[i].evnamebuf, "TSO");
2748
2749 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC,
2750 NULL, sc->queues[i].evnamebuf,
2751 "Queue Bytes Received");
2752 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC,
2753 NULL, sc->queues[i].evnamebuf,
2754 "Queue Packets Received");
2755 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC,
2756 NULL, sc->queues[i].evnamebuf, "Rx no mbuf");
2757 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC,
2758 NULL, sc->queues[i].evnamebuf, "Rx discarded");
2759 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC,
2760 NULL, sc->queues[i].evnamebuf, "Copied RX Frames");
2761 #ifdef LRO
2762 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
2763 CTLFLAG_RD, &lro->lro_queued, 0,
2764 "LRO Queued");
2765 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
2766 CTLFLAG_RD, &lro->lro_flushed, 0,
2767 "LRO Flushed");
2768 #endif /* LRO */
2769 }
2770
2771 /* MAC stats get their own sub node */
2772
2773 snprintf(stats->namebuf,
2774 sizeof(stats->namebuf), "%s MAC Statistics", xname);
2775
2776 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL,
2777 stats->namebuf, "rx csum offload - IP");
2778 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL,
2779 stats->namebuf, "rx csum offload - L4");
2780 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL,
2781 stats->namebuf, "rx csum offload - IP bad");
2782 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL,
2783 stats->namebuf, "rx csum offload - L4 bad");
2784
2785 /* Packet Reception Stats */
2786 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL,
2787 xname, "Good Packets Received");
2788 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL,
2789 xname, "Good Octets Received");
2790 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL,
2791 xname, "Multicast Packets Received");
2792 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL,
2793 xname, "Good Packets Transmitted");
2794 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL,
2795 xname, "Good Octets Transmitted");
2796
2797 /* Mailbox Stats */
2798 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL,
2799 xname, "message TXs");
2800 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL,
2801 xname, "message RXs");
2802 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL,
2803 xname, "ACKs");
2804 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL,
2805 xname, "REQs");
2806 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL,
2807 xname, "RSTs");
2808
2809 } /* ixv_add_stats_sysctls */
2810
2811 static void
ixv_clear_evcnt(struct ixgbe_softc * sc)2812 ixv_clear_evcnt(struct ixgbe_softc *sc)
2813 {
2814 struct tx_ring *txr = sc->tx_rings;
2815 struct rx_ring *rxr = sc->rx_rings;
2816 struct ixgbevf_hw_stats *stats = &sc->stats.vf;
2817 struct ixgbe_hw *hw = &sc->hw;
2818 int i;
2819
2820 /* Driver Statistics */
2821 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, 0);
2822 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, 0);
2823 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, 0);
2824 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, 0);
2825 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, 0);
2826 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, 0);
2827 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, 0);
2828 IXGBE_EVC_STORE(&sc->watchdog_events, 0);
2829 IXGBE_EVC_STORE(&sc->tso_err, 0);
2830 IXGBE_EVC_STORE(&sc->admin_irqev, 0);
2831 IXGBE_EVC_STORE(&sc->link_workev, 0);
2832
2833 for (i = 0; i < sc->num_queues; i++, rxr++, txr++) {
2834 IXGBE_EVC_STORE(&sc->queues[i].irqs, 0);
2835 IXGBE_EVC_STORE(&sc->queues[i].handleq, 0);
2836 IXGBE_EVC_STORE(&sc->queues[i].req, 0);
2837 IXGBE_EVC_STORE(&txr->total_packets, 0);
2838 #ifndef IXGBE_LEGACY_TX
2839 IXGBE_EVC_STORE(&txr->pcq_drops, 0);
2840 #endif
2841 IXGBE_EVC_STORE(&txr->no_desc_avail, 0);
2842 IXGBE_EVC_STORE(&txr->tso_tx, 0);
2843 txr->q_efbig_tx_dma_setup = 0;
2844 txr->q_mbuf_defrag_failed = 0;
2845 txr->q_efbig2_tx_dma_setup = 0;
2846 txr->q_einval_tx_dma_setup = 0;
2847 txr->q_other_tx_dma_setup = 0;
2848 txr->q_eagain_tx_dma_setup = 0;
2849 txr->q_enomem_tx_dma_setup = 0;
2850 txr->q_tso_err = 0;
2851
2852 IXGBE_EVC_STORE(&rxr->rx_packets, 0);
2853 IXGBE_EVC_STORE(&rxr->rx_bytes, 0);
2854 IXGBE_EVC_STORE(&rxr->rx_copies, 0);
2855 IXGBE_EVC_STORE(&rxr->no_mbuf, 0);
2856 IXGBE_EVC_STORE(&rxr->rx_discarded, 0);
2857 }
2858
2859 /* MAC stats get their own sub node */
2860
2861 IXGBE_EVC_STORE(&stats->ipcs, 0);
2862 IXGBE_EVC_STORE(&stats->l4cs, 0);
2863 IXGBE_EVC_STORE(&stats->ipcs_bad, 0);
2864 IXGBE_EVC_STORE(&stats->l4cs_bad, 0);
2865
2866 /*
2867 * Packet Reception Stats.
2868 * Call ixv_init_stats() to save last VF counters' values.
2869 */
2870 ixv_init_stats(sc);
2871 IXGBE_EVC_STORE(&stats->vfgprc, 0);
2872 IXGBE_EVC_STORE(&stats->vfgorc, 0);
2873 IXGBE_EVC_STORE(&stats->vfmprc, 0);
2874 IXGBE_EVC_STORE(&stats->vfgptc, 0);
2875 IXGBE_EVC_STORE(&stats->vfgotc, 0);
2876
2877 /* Mailbox Stats */
2878 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_tx, 0);
2879 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_rx, 0);
2880 IXGBE_EVC_STORE(&hw->mbx.stats.acks, 0);
2881 IXGBE_EVC_STORE(&hw->mbx.stats.reqs, 0);
2882 IXGBE_EVC_STORE(&hw->mbx.stats.rsts, 0);
2883
2884 } /* ixv_clear_evcnt */
2885
2886 #define PRINTQS(sc, regname) \
2887 do { \
2888 struct ixgbe_hw *_hw = &(sc)->hw; \
2889 int _i; \
2890 \
2891 printf("%s: %s", device_xname((sc)->dev), #regname); \
2892 for (_i = 0; _i < (sc)->num_queues; _i++) { \
2893 printf((_i == 0) ? "\t" : " "); \
2894 printf("%08x", IXGBE_READ_REG(_hw, \
2895 IXGBE_##regname(_i))); \
2896 } \
2897 printf("\n"); \
2898 } while (0)
2899
2900 /************************************************************************
2901 * ixv_print_debug_info
2902 *
2903 * Provides a way to take a look at important statistics
2904 * maintained by the driver and hardware.
2905 ************************************************************************/
2906 static void
ixv_print_debug_info(struct ixgbe_softc * sc)2907 ixv_print_debug_info(struct ixgbe_softc *sc)
2908 {
2909 device_t dev = sc->dev;
2910 struct ixgbe_hw *hw = &sc->hw;
2911 int i;
2912
2913 device_printf(dev, "queue:");
2914 for (i = 0; i < sc->num_queues; i++) {
2915 printf((i == 0) ? "\t" : " ");
2916 printf("%8d", i);
2917 }
2918 printf("\n");
2919 PRINTQS(sc, VFRDBAL);
2920 PRINTQS(sc, VFRDBAH);
2921 PRINTQS(sc, VFRDLEN);
2922 PRINTQS(sc, VFSRRCTL);
2923 PRINTQS(sc, VFRDH);
2924 PRINTQS(sc, VFRDT);
2925 PRINTQS(sc, VFRXDCTL);
2926
2927 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIMS));
2928 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAM));
2929 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAC));
2930 } /* ixv_print_debug_info */
2931
2932 /************************************************************************
2933 * ixv_sysctl_debug
2934 ************************************************************************/
2935 static int
ixv_sysctl_debug(SYSCTLFN_ARGS)2936 ixv_sysctl_debug(SYSCTLFN_ARGS)
2937 {
2938 struct sysctlnode node = *rnode;
2939 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
2940 int error, result = 0;
2941
2942 node.sysctl_data = &result;
2943 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2944
2945 if (error || newp == NULL)
2946 return error;
2947
2948 if (result == 1)
2949 ixv_print_debug_info(sc);
2950
2951 return 0;
2952 } /* ixv_sysctl_debug */
2953
2954 /************************************************************************
2955 * ixv_sysctl_rx_copy_len
2956 ************************************************************************/
2957 static int
ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS)2958 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS)
2959 {
2960 struct sysctlnode node = *rnode;
2961 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
2962 int error;
2963 int result = sc->rx_copy_len;
2964
2965 node.sysctl_data = &result;
2966 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2967
2968 if (error || newp == NULL)
2969 return error;
2970
2971 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX))
2972 return EINVAL;
2973
2974 sc->rx_copy_len = result;
2975
2976 return 0;
2977 } /* ixv_sysctl_rx_copy_len */
2978
2979 /************************************************************************
2980 * ixv_sysctl_tx_process_limit
2981 ************************************************************************/
2982 static int
ixv_sysctl_tx_process_limit(SYSCTLFN_ARGS)2983 ixv_sysctl_tx_process_limit(SYSCTLFN_ARGS)
2984 {
2985 struct sysctlnode node = *rnode;
2986 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
2987 int error;
2988 int result = sc->tx_process_limit;
2989
2990 node.sysctl_data = &result;
2991 error = sysctl_lookup(SYSCTLFN_CALL(&node));
2992
2993 if (error || newp == NULL)
2994 return error;
2995
2996 if ((result <= 0) || (result > sc->num_tx_desc))
2997 return EINVAL;
2998
2999 sc->tx_process_limit = result;
3000
3001 return 0;
3002 } /* ixv_sysctl_tx_process_limit */
3003
3004 /************************************************************************
3005 * ixv_sysctl_rx_process_limit
3006 ************************************************************************/
3007 static int
ixv_sysctl_rx_process_limit(SYSCTLFN_ARGS)3008 ixv_sysctl_rx_process_limit(SYSCTLFN_ARGS)
3009 {
3010 struct sysctlnode node = *rnode;
3011 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
3012 int error;
3013 int result = sc->rx_process_limit;
3014
3015 node.sysctl_data = &result;
3016 error = sysctl_lookup(SYSCTLFN_CALL(&node));
3017
3018 if (error || newp == NULL)
3019 return error;
3020
3021 if ((result <= 0) || (result > sc->num_rx_desc))
3022 return EINVAL;
3023
3024 sc->rx_process_limit = result;
3025
3026 return 0;
3027 } /* ixv_sysctl_rx_process_limit */
3028
3029 /************************************************************************
3030 * ixv_init_device_features
3031 ************************************************************************/
3032 static void
ixv_init_device_features(struct ixgbe_softc * sc)3033 ixv_init_device_features(struct ixgbe_softc *sc)
3034 {
3035 sc->feat_cap = IXGBE_FEATURE_NETMAP
3036 | IXGBE_FEATURE_VF
3037 | IXGBE_FEATURE_RSS
3038 | IXGBE_FEATURE_LEGACY_TX;
3039
3040 /* A tad short on feature flags for VFs, atm. */
3041 switch (sc->hw.mac.type) {
3042 case ixgbe_mac_82599_vf:
3043 break;
3044 case ixgbe_mac_X540_vf:
3045 break;
3046 case ixgbe_mac_X550_vf:
3047 case ixgbe_mac_X550EM_x_vf:
3048 case ixgbe_mac_X550EM_a_vf:
3049 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD;
3050 break;
3051 default:
3052 break;
3053 }
3054
3055 /* Enabled by default... */
3056 /* Is a virtual function (VF) */
3057 if (sc->feat_cap & IXGBE_FEATURE_VF)
3058 sc->feat_en |= IXGBE_FEATURE_VF;
3059 /* Netmap */
3060 if (sc->feat_cap & IXGBE_FEATURE_NETMAP)
3061 sc->feat_en |= IXGBE_FEATURE_NETMAP;
3062 /* Receive-Side Scaling (RSS) */
3063 if (sc->feat_cap & IXGBE_FEATURE_RSS)
3064 sc->feat_en |= IXGBE_FEATURE_RSS;
3065 /* Needs advanced context descriptor regardless of offloads req'd */
3066 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD)
3067 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD;
3068
3069 /* Enabled via sysctl... */
3070 /* Legacy (single queue) transmit */
3071 if ((sc->feat_cap & IXGBE_FEATURE_LEGACY_TX) &&
3072 ixv_enable_legacy_tx)
3073 sc->feat_en |= IXGBE_FEATURE_LEGACY_TX;
3074 } /* ixv_init_device_features */
3075
3076 /************************************************************************
3077 * ixv_shutdown - Shutdown entry point
3078 ************************************************************************/
3079 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */
3080 static int
3081 ixv_shutdown(device_t dev)
3082 {
3083 struct ixgbe_softc *sc = device_private(dev);
3084 IXGBE_CORE_LOCK(sc);
3085 ixv_stop_locked(sc);
3086 IXGBE_CORE_UNLOCK(sc);
3087
3088 return (0);
3089 } /* ixv_shutdown */
3090 #endif
3091
3092 static int
ixv_ifflags_cb(struct ethercom * ec)3093 ixv_ifflags_cb(struct ethercom *ec)
3094 {
3095 struct ifnet *ifp = &ec->ec_if;
3096 struct ixgbe_softc *sc = ifp->if_softc;
3097 u_short saved_flags;
3098 u_short change;
3099 int rv = 0;
3100
3101 IXGBE_CORE_LOCK(sc);
3102
3103 saved_flags = sc->if_flags;
3104 change = ifp->if_flags ^ sc->if_flags;
3105 if (change != 0)
3106 sc->if_flags = ifp->if_flags;
3107
3108 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) {
3109 rv = ENETRESET;
3110 goto out;
3111 } else if ((change & IFF_PROMISC) != 0) {
3112 rv = ixv_set_rxfilter(sc);
3113 if (rv != 0) {
3114 /* Restore previous */
3115 sc->if_flags = saved_flags;
3116 goto out;
3117 }
3118 }
3119
3120 /* Check for ec_capenable. */
3121 change = ec->ec_capenable ^ sc->ec_capenable;
3122 sc->ec_capenable = ec->ec_capenable;
3123 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING
3124 | ETHERCAP_VLAN_HWFILTER)) != 0) {
3125 rv = ENETRESET;
3126 goto out;
3127 }
3128
3129 /*
3130 * Special handling is not required for ETHERCAP_VLAN_MTU.
3131 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header.
3132 */
3133
3134 /* Set up VLAN support and filter */
3135 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0)
3136 rv = ixv_setup_vlan_support(sc);
3137
3138 out:
3139 IXGBE_CORE_UNLOCK(sc);
3140
3141 return rv;
3142 }
3143
3144
3145 /************************************************************************
3146 * ixv_ioctl - Ioctl entry point
3147 *
3148 * Called when the user wants to configure the interface.
3149 *
3150 * return 0 on success, positive on failure
3151 ************************************************************************/
3152 static int
ixv_ioctl(struct ifnet * ifp,u_long command,void * data)3153 ixv_ioctl(struct ifnet *ifp, u_long command, void *data)
3154 {
3155 struct ixgbe_softc *sc = ifp->if_softc;
3156 struct ixgbe_hw *hw = &sc->hw;
3157 struct ifcapreq *ifcr = data;
3158 int error;
3159 int l4csum_en;
3160 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
3161 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;
3162
3163 switch (command) {
3164 case SIOCSIFFLAGS:
3165 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
3166 break;
3167 case SIOCADDMULTI: {
3168 struct ether_multi *enm;
3169 struct ether_multistep step;
3170 struct ethercom *ec = &sc->osdep.ec;
3171 bool overflow = false;
3172 int mcnt = 0;
3173
3174 /*
3175 * Check the number of multicast address. If it exceeds,
3176 * return ENOSPC.
3177 * Update this code when we support API 1.3.
3178 */
3179 ETHER_LOCK(ec);
3180 ETHER_FIRST_MULTI(step, ec, enm);
3181 while (enm != NULL) {
3182 mcnt++;
3183
3184 /*
3185 * This code is before adding, so one room is required
3186 * at least.
3187 */
3188 if (mcnt > (IXGBE_MAX_VF_MC - 1)) {
3189 overflow = true;
3190 break;
3191 }
3192 ETHER_NEXT_MULTI(step, enm);
3193 }
3194 ETHER_UNLOCK(ec);
3195 error = 0;
3196 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) {
3197 error = hw->mac.ops.update_xcast_mode(hw,
3198 IXGBEVF_XCAST_MODE_ALLMULTI);
3199 if (error == IXGBE_ERR_NOT_TRUSTED) {
3200 device_printf(sc->dev,
3201 "this interface is not trusted\n");
3202 error = EPERM;
3203 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) {
3204 device_printf(sc->dev,
3205 "the PF doesn't support allmulti mode\n");
3206 error = EOPNOTSUPP;
3207 } else if (error) {
3208 device_printf(sc->dev,
3209 "number of Ethernet multicast addresses "
3210 "exceeds the limit (%d). error = %d\n",
3211 IXGBE_MAX_VF_MC, error);
3212 error = ENOSPC;
3213 } else
3214 ec->ec_flags |= ETHER_F_ALLMULTI;
3215 }
3216 if (error)
3217 return error;
3218 }
3219 /*FALLTHROUGH*/
3220 case SIOCDELMULTI:
3221 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
3222 break;
3223 case SIOCSIFMEDIA:
3224 case SIOCGIFMEDIA:
3225 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
3226 break;
3227 case SIOCSIFCAP:
3228 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
3229 break;
3230 case SIOCSIFMTU:
3231 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
3232 break;
3233 case SIOCZIFDATA:
3234 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)");
3235 ixv_update_stats(sc);
3236 ixv_clear_evcnt(sc);
3237 break;
3238 default:
3239 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command);
3240 break;
3241 }
3242
3243 switch (command) {
3244 case SIOCSIFCAP:
3245 /* Layer-4 Rx checksum offload has to be turned on and
3246 * off as a unit.
3247 */
3248 l4csum_en = ifcr->ifcr_capenable & l4csum;
3249 if (l4csum_en != l4csum && l4csum_en != 0)
3250 return EINVAL;
3251 /*FALLTHROUGH*/
3252 case SIOCADDMULTI:
3253 case SIOCDELMULTI:
3254 case SIOCSIFFLAGS:
3255 case SIOCSIFMTU:
3256 default:
3257 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET)
3258 return error;
3259 if ((ifp->if_flags & IFF_RUNNING) == 0)
3260 ;
3261 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) {
3262 IXGBE_CORE_LOCK(sc);
3263 ixv_init_locked(sc);
3264 IXGBE_CORE_UNLOCK(sc);
3265 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) {
3266 /*
3267 * Multicast list has changed; set the hardware filter
3268 * accordingly.
3269 */
3270 IXGBE_CORE_LOCK(sc);
3271 ixv_disable_intr(sc);
3272 ixv_set_rxfilter(sc);
3273 ixv_enable_intr(sc);
3274 IXGBE_CORE_UNLOCK(sc);
3275 }
3276 return 0;
3277 }
3278 } /* ixv_ioctl */
3279
3280 /************************************************************************
3281 * ixv_init
3282 ************************************************************************/
3283 static int
ixv_init(struct ifnet * ifp)3284 ixv_init(struct ifnet *ifp)
3285 {
3286 struct ixgbe_softc *sc = ifp->if_softc;
3287
3288 IXGBE_CORE_LOCK(sc);
3289 ixv_init_locked(sc);
3290 IXGBE_CORE_UNLOCK(sc);
3291
3292 return 0;
3293 } /* ixv_init */
3294
3295 /************************************************************************
3296 * ixv_handle_que
3297 ************************************************************************/
3298 static void
ixv_handle_que(void * context)3299 ixv_handle_que(void *context)
3300 {
3301 struct ix_queue *que = context;
3302 struct ixgbe_softc *sc = que->sc;
3303 struct tx_ring *txr = que->txr;
3304 struct ifnet *ifp = sc->ifp;
3305 bool more;
3306
3307 IXGBE_EVC_ADD(&que->handleq, 1);
3308
3309 if (ifp->if_flags & IFF_RUNNING) {
3310 IXGBE_TX_LOCK(txr);
3311 more = ixgbe_txeof(txr);
3312 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX))
3313 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq))
3314 ixgbe_mq_start_locked(ifp, txr);
3315 /* Only for queue 0 */
3316 /* NetBSD still needs this for CBQ */
3317 if ((&sc->queues[0] == que)
3318 && (!ixgbe_legacy_ring_empty(ifp, NULL)))
3319 ixgbe_legacy_start_locked(ifp, txr);
3320 IXGBE_TX_UNLOCK(txr);
3321 more |= ixgbe_rxeof(que);
3322 if (more) {
3323 IXGBE_EVC_ADD(&que->req, 1);
3324 if (sc->txrx_use_workqueue) {
3325 /*
3326 * "enqueued flag" is not required here
3327 * the same as ixg(4). See ixgbe_msix_que().
3328 */
3329 workqueue_enqueue(sc->que_wq,
3330 &que->wq_cookie, curcpu());
3331 } else
3332 softint_schedule(que->que_si);
3333 return;
3334 }
3335 }
3336
3337 /* Re-enable this interrupt */
3338 ixv_enable_queue(sc, que->msix);
3339
3340 return;
3341 } /* ixv_handle_que */
3342
3343 /************************************************************************
3344 * ixv_handle_que_work
3345 ************************************************************************/
3346 static void
ixv_handle_que_work(struct work * wk,void * context)3347 ixv_handle_que_work(struct work *wk, void *context)
3348 {
3349 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie);
3350
3351 /*
3352 * "enqueued flag" is not required here the same as ixg(4).
3353 * See ixgbe_msix_que().
3354 */
3355 ixv_handle_que(que);
3356 }
3357
3358 /************************************************************************
3359 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers
3360 ************************************************************************/
3361 static int
ixv_allocate_msix(struct ixgbe_softc * sc,const struct pci_attach_args * pa)3362 ixv_allocate_msix(struct ixgbe_softc *sc, const struct pci_attach_args *pa)
3363 {
3364 device_t dev = sc->dev;
3365 struct ix_queue *que = sc->queues;
3366 struct tx_ring *txr = sc->tx_rings;
3367 int error, msix_ctrl, rid, vector = 0;
3368 pci_chipset_tag_t pc;
3369 pcitag_t tag;
3370 char intrbuf[PCI_INTRSTR_LEN];
3371 char wqname[MAXCOMLEN];
3372 char intr_xname[32];
3373 const char *intrstr = NULL;
3374 kcpuset_t *affinity;
3375 int cpu_id = 0;
3376
3377 pc = sc->osdep.pc;
3378 tag = sc->osdep.tag;
3379
3380 sc->osdep.nintrs = sc->num_queues + 1;
3381 if (pci_msix_alloc_exact(pa, &sc->osdep.intrs,
3382 sc->osdep.nintrs) != 0) {
3383 aprint_error_dev(dev,
3384 "failed to allocate MSI-X interrupt\n");
3385 return (ENXIO);
3386 }
3387
3388 kcpuset_create(&affinity, false);
3389 for (int i = 0; i < sc->num_queues; i++, vector++, que++, txr++) {
3390 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
3391 device_xname(dev), i);
3392 intrstr = pci_intr_string(pc, sc->osdep.intrs[i], intrbuf,
3393 sizeof(intrbuf));
3394 pci_intr_setattr(pc, &sc->osdep.intrs[i], PCI_INTR_MPSAFE,
3395 true);
3396
3397 /* Set the handler function */
3398 que->res = sc->osdep.ihs[i] = pci_intr_establish_xname(pc,
3399 sc->osdep.intrs[i], IPL_NET, ixv_msix_que, que,
3400 intr_xname);
3401 if (que->res == NULL) {
3402 pci_intr_release(pc, sc->osdep.intrs,
3403 sc->osdep.nintrs);
3404 aprint_error_dev(dev,
3405 "Failed to register QUE handler\n");
3406 kcpuset_destroy(affinity);
3407 return (ENXIO);
3408 }
3409 que->msix = vector;
3410 sc->active_queues |= (u64)(1 << que->msix);
3411
3412 cpu_id = i;
3413 /* Round-robin affinity */
3414 kcpuset_zero(affinity);
3415 kcpuset_set(affinity, cpu_id % ncpu);
3416 error = interrupt_distribute(sc->osdep.ihs[i], affinity, NULL);
3417 aprint_normal_dev(dev, "for TX/RX, interrupting at %s",
3418 intrstr);
3419 if (error == 0)
3420 aprint_normal(", bound queue %d to cpu %d\n",
3421 i, cpu_id % ncpu);
3422 else
3423 aprint_normal("\n");
3424
3425 #ifndef IXGBE_LEGACY_TX
3426 txr->txr_si
3427 = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
3428 ixgbe_deferred_mq_start, txr);
3429 #endif
3430 que->que_si
3431 = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
3432 ixv_handle_que, que);
3433 if (que->que_si == NULL) {
3434 aprint_error_dev(dev,
3435 "could not establish software interrupt\n");
3436 }
3437 }
3438 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev));
3439 error = workqueue_create(&sc->txr_wq, wqname,
3440 ixgbe_deferred_mq_start_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET,
3441 WQ_PERCPU | WQ_MPSAFE);
3442 if (error) {
3443 aprint_error_dev(dev,
3444 "couldn't create workqueue for deferred Tx\n");
3445 }
3446 sc->txr_wq_enqueued = percpu_alloc(sizeof(u_int));
3447
3448 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev));
3449 error = workqueue_create(&sc->que_wq, wqname,
3450 ixv_handle_que_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET,
3451 WQ_PERCPU | WQ_MPSAFE);
3452 if (error) {
3453 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n");
3454 }
3455
3456 /* and Mailbox */
3457 cpu_id++;
3458 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev));
3459 sc->vector = vector;
3460 intrstr = pci_intr_string(pc, sc->osdep.intrs[vector], intrbuf,
3461 sizeof(intrbuf));
3462 pci_intr_setattr(pc, &sc->osdep.intrs[vector], PCI_INTR_MPSAFE, true);
3463
3464 /* Set the mbx handler function */
3465 sc->osdep.ihs[vector] = pci_intr_establish_xname(pc,
3466 sc->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, sc, intr_xname);
3467 if (sc->osdep.ihs[vector] == NULL) {
3468 aprint_error_dev(dev, "Failed to register LINK handler\n");
3469 kcpuset_destroy(affinity);
3470 return (ENXIO);
3471 }
3472 /* Round-robin affinity */
3473 kcpuset_zero(affinity);
3474 kcpuset_set(affinity, cpu_id % ncpu);
3475 error = interrupt_distribute(sc->osdep.ihs[vector], affinity,
3476 NULL);
3477
3478 aprint_normal_dev(dev,
3479 "for link, interrupting at %s", intrstr);
3480 if (error == 0)
3481 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu);
3482 else
3483 aprint_normal("\n");
3484
3485 /* Tasklets for Mailbox */
3486 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev));
3487 error = workqueue_create(&sc->admin_wq, wqname,
3488 ixv_handle_admin, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE);
3489 if (error) {
3490 aprint_error_dev(dev,
3491 "could not create admin workqueue (%d)\n", error);
3492 goto err_out;
3493 }
3494
3495 /*
3496 * Due to a broken design QEMU will fail to properly
3497 * enable the guest for MSI-X unless the vectors in
3498 * the table are all set up, so we must rewrite the
3499 * ENABLE in the MSI-X control register again at this
3500 * point to cause it to successfully initialize us.
3501 */
3502 if (sc->hw.mac.type == ixgbe_mac_82599_vf) {
3503 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL);
3504 rid += PCI_MSIX_CTL;
3505 msix_ctrl = pci_conf_read(pc, tag, rid);
3506 msix_ctrl |= PCI_MSIX_CTL_ENABLE;
3507 pci_conf_write(pc, tag, rid, msix_ctrl);
3508 }
3509
3510 kcpuset_destroy(affinity);
3511 return (0);
3512 err_out:
3513 kcpuset_destroy(affinity);
3514 ixv_free_deferred_handlers(sc);
3515 ixv_free_pci_resources(sc);
3516 return (error);
3517 } /* ixv_allocate_msix */
3518
3519 /************************************************************************
3520 * ixv_configure_interrupts - Setup MSI-X resources
3521 *
3522 * Note: The VF device MUST use MSI-X, there is no fallback.
3523 ************************************************************************/
3524 static int
ixv_configure_interrupts(struct ixgbe_softc * sc)3525 ixv_configure_interrupts(struct ixgbe_softc *sc)
3526 {
3527 device_t dev = sc->dev;
3528 int want, queues, msgs;
3529
3530 /* Must have at least 2 MSI-X vectors */
3531 msgs = pci_msix_count(sc->osdep.pc, sc->osdep.tag);
3532 if (msgs < 2) {
3533 aprint_error_dev(dev, "MSIX config error\n");
3534 return (ENXIO);
3535 }
3536 msgs = MIN(msgs, IXG_MAX_NINTR);
3537
3538 /* Figure out a reasonable auto config value */
3539 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu;
3540
3541 if (ixv_num_queues != 0)
3542 queues = ixv_num_queues;
3543 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES))
3544 queues = IXGBE_VF_MAX_TX_QUEUES;
3545
3546 /*
3547 * Want vectors for the queues,
3548 * plus an additional for mailbox.
3549 */
3550 want = queues + 1;
3551 if (msgs >= want)
3552 msgs = want;
3553 else {
3554 aprint_error_dev(dev,
3555 "MSI-X Configuration Problem, "
3556 "%d vectors but %d queues wanted!\n", msgs, want);
3557 return -1;
3558 }
3559
3560 aprint_normal_dev(dev,
3561 "Using MSI-X interrupts with %d vectors\n", msgs);
3562 sc->num_queues = queues;
3563
3564 return (0);
3565 } /* ixv_configure_interrupts */
3566
3567
3568 /************************************************************************
3569 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts
3570 *
3571 * Done outside of interrupt context since the driver might sleep
3572 ************************************************************************/
3573 static void
ixv_handle_admin(struct work * wk,void * context)3574 ixv_handle_admin(struct work *wk, void *context)
3575 {
3576 struct ixgbe_softc *sc = context;
3577 struct ixgbe_hw *hw = &sc->hw;
3578
3579 IXGBE_CORE_LOCK(sc);
3580
3581 IXGBE_EVC_ADD(&sc->link_workev, 1);
3582 sc->hw.mac.ops.check_link(&sc->hw, &sc->link_speed,
3583 &sc->link_up, FALSE);
3584 ixv_update_link_status(sc);
3585
3586 sc->task_requests = 0;
3587 atomic_store_relaxed(&sc->admin_pending, 0);
3588
3589 /* Re-enable interrupts */
3590 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector));
3591
3592 IXGBE_CORE_UNLOCK(sc);
3593 } /* ixv_handle_admin */
3594
3595 /************************************************************************
3596 * ixv_check_link - Used in the local timer to poll for link changes
3597 ************************************************************************/
3598 static s32
ixv_check_link(struct ixgbe_softc * sc)3599 ixv_check_link(struct ixgbe_softc *sc)
3600 {
3601 s32 error;
3602
3603 KASSERT(mutex_owned(&sc->core_mtx));
3604
3605 sc->hw.mac.get_link_status = TRUE;
3606
3607 error = sc->hw.mac.ops.check_link(&sc->hw,
3608 &sc->link_speed, &sc->link_up, FALSE);
3609 ixv_update_link_status(sc);
3610
3611 return error;
3612 } /* ixv_check_link */
3613