1 /* $NetBSD: ixv.c,v 1.198 2024/07/10 03:26:30 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.198 2024/07/10 03:26:30 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #endif 44 45 #include "ixgbe.h" 46 47 /************************************************************************ 48 * Driver version 49 ************************************************************************/ 50 static const char ixv_driver_version[] = "2.0.1-k"; 51 /* XXX NetBSD: + 1.5.17 */ 52 53 /************************************************************************ 54 * PCI Device ID Table 55 * 56 * Used by probe to select devices to load on 57 * Last field stores an index into ixv_strings 58 * Last entry must be all 0s 59 * 60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 61 ************************************************************************/ 62 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 63 { 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 69 /* required last entry */ 70 {0, 0, 0, 0, 0} 71 }; 72 73 /************************************************************************ 74 * Table of branding strings 75 ************************************************************************/ 76 static const char *ixv_strings[] = { 77 "Intel(R) PRO/10GbE Virtual Function Network Driver" 78 }; 79 80 /********************************************************************* 81 * Function prototypes 82 *********************************************************************/ 83 static int ixv_probe(device_t, cfdata_t, void *); 84 static void ixv_attach(device_t, device_t, void *); 85 static int ixv_detach(device_t, int); 86 #if 0 87 static int ixv_shutdown(device_t); 88 #endif 89 static int ixv_ifflags_cb(struct ethercom *); 90 static int ixv_ioctl(struct ifnet *, u_long, void *); 91 static int ixv_init(struct ifnet *); 92 static void ixv_init_locked(struct ixgbe_softc *); 93 static void ixv_ifstop(struct ifnet *, int); 94 static void ixv_stop_locked(void *); 95 static void ixv_init_device_features(struct ixgbe_softc *); 96 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 97 static int ixv_media_change(struct ifnet *); 98 static int ixv_allocate_pci_resources(struct ixgbe_softc *, 99 const struct pci_attach_args *); 100 static void ixv_free_deferred_handlers(struct ixgbe_softc *); 101 static int ixv_allocate_msix(struct ixgbe_softc *, 102 const struct pci_attach_args *); 103 static int ixv_configure_interrupts(struct ixgbe_softc *); 104 static void ixv_free_pci_resources(struct ixgbe_softc *); 105 static void ixv_local_timer(void *); 106 static void ixv_handle_timer(struct work *, void *); 107 static int ixv_setup_interface(device_t, struct ixgbe_softc *); 108 static void ixv_schedule_admin_tasklet(struct ixgbe_softc *); 109 static int ixv_negotiate_api(struct ixgbe_softc *); 110 111 static void ixv_initialize_transmit_units(struct ixgbe_softc *); 112 static void ixv_initialize_receive_units(struct ixgbe_softc *); 113 static void ixv_initialize_rss_mapping(struct ixgbe_softc *); 114 static s32 ixv_check_link(struct ixgbe_softc *); 115 116 static void ixv_enable_intr(struct ixgbe_softc *); 117 static void ixv_disable_intr(struct ixgbe_softc *); 118 static int ixv_set_rxfilter(struct ixgbe_softc *); 119 static void ixv_update_link_status(struct ixgbe_softc *); 120 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 121 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8); 122 static void ixv_configure_ivars(struct ixgbe_softc *); 123 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 124 static void ixv_eitr_write(struct ixgbe_softc *, uint32_t, uint32_t); 125 126 static void ixv_setup_vlan_tagging(struct ixgbe_softc *); 127 static int ixv_setup_vlan_support(struct ixgbe_softc *); 128 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 129 static int ixv_register_vlan(struct ixgbe_softc *, u16); 130 static int ixv_unregister_vlan(struct ixgbe_softc *, u16); 131 132 static void ixv_add_device_sysctls(struct ixgbe_softc *); 133 static void ixv_init_stats(struct ixgbe_softc *); 134 static void ixv_update_stats(struct ixgbe_softc *); 135 static void ixv_add_stats_sysctls(struct ixgbe_softc *); 136 static void ixv_clear_evcnt(struct ixgbe_softc *); 137 138 /* Sysctl handlers */ 139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_tx_process_limit(SYSCTLFN_PROTO); 147 static int ixv_sysctl_rx_process_limit(SYSCTLFN_PROTO); 148 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 149 150 /* The MSI-X Interrupt handlers */ 151 static int ixv_msix_que(void *); 152 static int ixv_msix_mbx(void *); 153 154 /* Event handlers running on workqueue */ 155 static void ixv_handle_que(void *); 156 157 /* Deferred workqueue handlers */ 158 static void ixv_handle_admin(struct work *, void *); 159 static void ixv_handle_que_work(struct work *, void *); 160 161 const struct sysctlnode *ixv_sysctl_instance(struct ixgbe_softc *); 162 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 163 164 /************************************************************************ 165 * NetBSD Device Interface Entry Points 166 ************************************************************************/ 167 CFATTACH_DECL3_NEW(ixv, sizeof(struct ixgbe_softc), 168 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 169 DVF_DETACH_SHUTDOWN); 170 171 #if 0 172 static driver_t ixv_driver = { 173 "ixv", ixv_methods, sizeof(struct ixgbe_softc), 174 }; 175 176 devclass_t ixv_devclass; 177 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 178 MODULE_DEPEND(ixv, pci, 1, 1, 1); 179 MODULE_DEPEND(ixv, ether, 1, 1, 1); 180 #endif 181 182 /* 183 * TUNEABLE PARAMETERS: 184 */ 185 186 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 187 static int ixv_num_queues = 0; 188 #define TUNABLE_INT(__x, __y) 189 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 190 191 /* 192 * AIM: Adaptive Interrupt Moderation 193 * which means that the interrupt rate 194 * is varied over time based on the 195 * traffic for that interrupt vector 196 */ 197 static bool ixv_enable_aim = false; 198 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 199 200 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 201 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 202 203 /* How many packets rxeof tries to clean at a time */ 204 static int ixv_rx_process_limit = 256; 205 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 206 207 /* How many packets txeof tries to clean at a time */ 208 static int ixv_tx_process_limit = 256; 209 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 210 211 /* Which packet processing uses workqueue or softint */ 212 static bool ixv_txrx_workqueue = false; 213 214 /* 215 * Number of TX descriptors per ring, 216 * setting higher than RX as this seems 217 * the better performing choice. 218 */ 219 static int ixv_txd = DEFAULT_TXD; 220 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 221 222 /* Number of RX descriptors per ring */ 223 static int ixv_rxd = DEFAULT_RXD; 224 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 225 226 /* Legacy Transmit (single queue) */ 227 static int ixv_enable_legacy_tx = 0; 228 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 229 230 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 231 232 #if 0 233 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 234 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 235 #endif 236 237 /************************************************************************ 238 * ixv_probe - Device identification routine 239 * 240 * Determines if the driver should be loaded on 241 * adapter based on its PCI vendor/device ID. 242 * 243 * return BUS_PROBE_DEFAULT on success, positive on failure 244 ************************************************************************/ 245 static int 246 ixv_probe(device_t dev, cfdata_t cf, void *aux) 247 { 248 #ifdef __HAVE_PCI_MSI_MSIX 249 const struct pci_attach_args *pa = aux; 250 251 return (ixv_lookup(pa) != NULL) ? 1 : 0; 252 #else 253 return 0; 254 #endif 255 } /* ixv_probe */ 256 257 static const ixgbe_vendor_info_t * 258 ixv_lookup(const struct pci_attach_args *pa) 259 { 260 const ixgbe_vendor_info_t *ent; 261 pcireg_t subid; 262 263 INIT_DEBUGOUT("ixv_lookup: begin"); 264 265 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 266 return NULL; 267 268 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 269 270 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 271 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 272 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 273 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 274 (ent->subvendor_id == 0)) && 275 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 276 (ent->subdevice_id == 0))) { 277 return ent; 278 } 279 } 280 281 return NULL; 282 } 283 284 /************************************************************************ 285 * ixv_attach - Device initialization routine 286 * 287 * Called when the driver is being loaded. 288 * Identifies the type of hardware, allocates all resources 289 * and initializes the hardware. 290 * 291 * return 0 on success, positive on failure 292 ************************************************************************/ 293 static void 294 ixv_attach(device_t parent, device_t dev, void *aux) 295 { 296 struct ixgbe_softc *sc; 297 struct ixgbe_hw *hw; 298 int error = 0; 299 pcireg_t id, subid; 300 const ixgbe_vendor_info_t *ent; 301 const struct pci_attach_args *pa = aux; 302 const char *apivstr; 303 const char *str; 304 char wqname[MAXCOMLEN]; 305 char buf[256]; 306 307 INIT_DEBUGOUT("ixv_attach: begin"); 308 309 /* 310 * Make sure BUSMASTER is set, on a VM under 311 * KVM it may not be and will break things. 312 */ 313 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 314 315 /* Allocate, clear, and link in our adapter structure */ 316 sc = device_private(dev); 317 sc->hw.back = sc; 318 sc->dev = dev; 319 hw = &sc->hw; 320 321 sc->init_locked = ixv_init_locked; 322 sc->stop_locked = ixv_stop_locked; 323 324 sc->osdep.pc = pa->pa_pc; 325 sc->osdep.tag = pa->pa_tag; 326 if (pci_dma64_available(pa)) 327 sc->osdep.dmat = pa->pa_dmat64; 328 else 329 sc->osdep.dmat = pa->pa_dmat; 330 sc->osdep.attached = false; 331 332 ent = ixv_lookup(pa); 333 334 KASSERT(ent != NULL); 335 336 aprint_normal(": %s, Version - %s\n", 337 ixv_strings[ent->index], ixv_driver_version); 338 339 /* Core Lock Init */ 340 IXGBE_CORE_LOCK_INIT(sc, device_xname(dev)); 341 342 /* Do base PCI setup - map BAR0 */ 343 if (ixv_allocate_pci_resources(sc, pa)) { 344 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 345 error = ENXIO; 346 goto err_out; 347 } 348 349 /* SYSCTL APIs */ 350 ixv_add_device_sysctls(sc); 351 352 /* Set up the timer callout and workqueue */ 353 callout_init(&sc->timer, CALLOUT_MPSAFE); 354 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 355 error = workqueue_create(&sc->timer_wq, wqname, 356 ixv_handle_timer, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE); 357 if (error) { 358 aprint_error_dev(dev, 359 "could not create timer workqueue (%d)\n", error); 360 goto err_out; 361 } 362 363 /* Save off the information about this board */ 364 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 365 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 366 hw->vendor_id = PCI_VENDOR(id); 367 hw->device_id = PCI_PRODUCT(id); 368 hw->revision_id = 369 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 370 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 371 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 372 373 /* A subset of set_mac_type */ 374 switch (hw->device_id) { 375 case IXGBE_DEV_ID_82599_VF: 376 hw->mac.type = ixgbe_mac_82599_vf; 377 str = "82599 VF"; 378 break; 379 case IXGBE_DEV_ID_X540_VF: 380 hw->mac.type = ixgbe_mac_X540_vf; 381 str = "X540 VF"; 382 break; 383 case IXGBE_DEV_ID_X550_VF: 384 hw->mac.type = ixgbe_mac_X550_vf; 385 str = "X550 VF"; 386 break; 387 case IXGBE_DEV_ID_X550EM_X_VF: 388 hw->mac.type = ixgbe_mac_X550EM_x_vf; 389 str = "X550EM X VF"; 390 break; 391 case IXGBE_DEV_ID_X550EM_A_VF: 392 hw->mac.type = ixgbe_mac_X550EM_a_vf; 393 str = "X550EM A VF"; 394 break; 395 default: 396 /* Shouldn't get here since probe succeeded */ 397 aprint_error_dev(dev, "Unknown device ID!\n"); 398 error = ENXIO; 399 goto err_out; 400 break; 401 } 402 aprint_normal_dev(dev, "device %s\n", str); 403 404 ixv_init_device_features(sc); 405 406 /* Initialize the shared code */ 407 error = ixgbe_init_ops_vf(hw); 408 if (error) { 409 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 410 error = EIO; 411 goto err_out; 412 } 413 414 /* Setup the mailbox */ 415 ixgbe_init_mbx_params_vf(hw); 416 417 /* Set the right number of segments */ 418 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 419 sc->num_segs = IXGBE_SCATTER_DEFAULT; 420 421 /* Reset mbox api to 1.0 */ 422 error = hw->mac.ops.reset_hw(hw); 423 if (error == IXGBE_ERR_RESET_FAILED) 424 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 425 else if (error) 426 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 427 error); 428 if (error) { 429 error = EIO; 430 goto err_out; 431 } 432 433 error = hw->mac.ops.init_hw(hw); 434 if (error) { 435 aprint_error_dev(dev, "...init_hw() failed!\n"); 436 error = EIO; 437 goto err_out; 438 } 439 440 /* Negotiate mailbox API version */ 441 error = ixv_negotiate_api(sc); 442 if (error) 443 aprint_normal_dev(dev, 444 "MBX API negotiation failed during attach!\n"); 445 switch (hw->api_version) { 446 case ixgbe_mbox_api_10: 447 apivstr = "1.0"; 448 break; 449 case ixgbe_mbox_api_20: 450 apivstr = "2.0"; 451 break; 452 case ixgbe_mbox_api_11: 453 apivstr = "1.1"; 454 break; 455 case ixgbe_mbox_api_12: 456 apivstr = "1.2"; 457 break; 458 case ixgbe_mbox_api_13: 459 apivstr = "1.3"; 460 break; 461 case ixgbe_mbox_api_14: 462 apivstr = "1.4"; 463 break; 464 case ixgbe_mbox_api_15: 465 apivstr = "1.5"; 466 break; 467 default: 468 apivstr = "unknown"; 469 break; 470 } 471 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 472 473 /* If no mac address was assigned, make a random one */ 474 if (!ixv_check_ether_addr(hw->mac.addr)) { 475 u8 addr[ETHER_ADDR_LEN]; 476 uint64_t rndval = cprng_strong64(); 477 478 memcpy(addr, &rndval, sizeof(addr)); 479 addr[0] &= 0xFE; 480 addr[0] |= 0x02; 481 bcopy(addr, hw->mac.addr, sizeof(addr)); 482 } 483 484 /* Register for VLAN events */ 485 ether_set_vlan_cb(&sc->osdep.ec, ixv_vlan_cb); 486 487 /* Do descriptor calc and sanity checks */ 488 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 489 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 490 aprint_error_dev(dev, "Invalid TX ring size (%d). " 491 "It must be between %d and %d, " 492 "inclusive, and must be a multiple of %zu. " 493 "Using default value of %d instead.\n", 494 ixv_txd, MIN_TXD, MAX_TXD, 495 DBA_ALIGN / sizeof(union ixgbe_adv_tx_desc), 496 DEFAULT_TXD); 497 sc->num_tx_desc = DEFAULT_TXD; 498 } else 499 sc->num_tx_desc = ixv_txd; 500 501 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 502 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 503 aprint_error_dev(dev, "Invalid RX ring size (%d). " 504 "It must be between %d and %d, " 505 "inclusive, and must be a multiple of %zu. " 506 "Using default value of %d instead.\n", 507 ixv_rxd, MIN_RXD, MAX_RXD, 508 DBA_ALIGN / sizeof(union ixgbe_adv_rx_desc), 509 DEFAULT_RXD); 510 sc->num_rx_desc = DEFAULT_RXD; 511 } else 512 sc->num_rx_desc = ixv_rxd; 513 514 /* Sysctls for limiting the amount of work done in the taskqueues */ 515 sc->rx_process_limit 516 = (ixv_rx_process_limit <= sc->num_rx_desc) 517 ? ixv_rx_process_limit : sc->num_rx_desc; 518 sc->tx_process_limit 519 = (ixv_tx_process_limit <= sc->num_tx_desc) 520 ? ixv_tx_process_limit : sc->num_tx_desc; 521 522 /* Set default high limit of copying mbuf in rxeof */ 523 sc->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 524 525 /* Setup MSI-X */ 526 error = ixv_configure_interrupts(sc); 527 if (error) 528 goto err_out; 529 530 /* Allocate our TX/RX Queues */ 531 if (ixgbe_allocate_queues(sc)) { 532 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 533 error = ENOMEM; 534 goto err_out; 535 } 536 537 /* hw.ix defaults init */ 538 sc->enable_aim = ixv_enable_aim; 539 sc->max_interrupt_rate = ixv_max_interrupt_rate; 540 541 sc->txrx_use_workqueue = ixv_txrx_workqueue; 542 543 error = ixv_allocate_msix(sc, pa); 544 if (error) { 545 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 546 goto err_late; 547 } 548 549 /* Setup OS specific network interface */ 550 error = ixv_setup_interface(dev, sc); 551 if (error != 0) { 552 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 553 goto err_late; 554 } 555 556 /* Allocate multicast array memory */ 557 sc->mta = malloc(sizeof(*sc->mta) * 558 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 559 560 /* Check if VF was disabled by PF */ 561 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 562 if (error) { 563 /* PF is not capable of controlling VF state. Enable the link. */ 564 sc->link_enabled = TRUE; 565 } 566 567 /* Do the stats setup */ 568 ixv_init_stats(sc); 569 ixv_add_stats_sysctls(sc); 570 571 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 572 ixgbe_netmap_attach(sc); 573 574 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_cap); 575 aprint_verbose_dev(dev, "feature cap %s\n", buf); 576 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_en); 577 aprint_verbose_dev(dev, "feature ena %s\n", buf); 578 579 INIT_DEBUGOUT("ixv_attach: end"); 580 sc->osdep.attached = true; 581 582 return; 583 584 err_late: 585 ixgbe_free_queues(sc); 586 err_out: 587 ixv_free_pci_resources(sc); 588 IXGBE_CORE_LOCK_DESTROY(sc); 589 590 return; 591 } /* ixv_attach */ 592 593 /************************************************************************ 594 * ixv_detach - Device removal routine 595 * 596 * Called when the driver is being removed. 597 * Stops the adapter and deallocates all the resources 598 * that were allocated for driver operation. 599 * 600 * return 0 on success, positive on failure 601 ************************************************************************/ 602 static int 603 ixv_detach(device_t dev, int flags) 604 { 605 struct ixgbe_softc *sc = device_private(dev); 606 struct ixgbe_hw *hw = &sc->hw; 607 struct tx_ring *txr = sc->tx_rings; 608 struct rx_ring *rxr = sc->rx_rings; 609 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 610 611 INIT_DEBUGOUT("ixv_detach: begin"); 612 if (sc->osdep.attached == false) 613 return 0; 614 615 /* Stop the interface. Callouts are stopped in it. */ 616 ixv_ifstop(sc->ifp, 1); 617 618 if (VLAN_ATTACHED(&sc->osdep.ec) && 619 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 620 aprint_error_dev(dev, "VLANs in use, detach first\n"); 621 return EBUSY; 622 } 623 624 ether_ifdetach(sc->ifp); 625 callout_halt(&sc->timer, NULL); 626 ixv_free_deferred_handlers(sc); 627 628 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 629 netmap_detach(sc->ifp); 630 631 ixv_free_pci_resources(sc); 632 #if 0 /* XXX the NetBSD port is probably missing something here */ 633 bus_generic_detach(dev); 634 #endif 635 if_detach(sc->ifp); 636 ifmedia_fini(&sc->media); 637 if_percpuq_destroy(sc->ipq); 638 639 sysctl_teardown(&sc->sysctllog); 640 evcnt_detach(&sc->efbig_tx_dma_setup); 641 evcnt_detach(&sc->mbuf_defrag_failed); 642 evcnt_detach(&sc->efbig2_tx_dma_setup); 643 evcnt_detach(&sc->einval_tx_dma_setup); 644 evcnt_detach(&sc->other_tx_dma_setup); 645 evcnt_detach(&sc->eagain_tx_dma_setup); 646 evcnt_detach(&sc->enomem_tx_dma_setup); 647 evcnt_detach(&sc->watchdog_events); 648 evcnt_detach(&sc->tso_err); 649 evcnt_detach(&sc->admin_irqev); 650 evcnt_detach(&sc->link_workev); 651 652 txr = sc->tx_rings; 653 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 654 evcnt_detach(&sc->queues[i].irqs); 655 evcnt_detach(&sc->queues[i].handleq); 656 evcnt_detach(&sc->queues[i].req); 657 evcnt_detach(&txr->total_packets); 658 #ifndef IXGBE_LEGACY_TX 659 evcnt_detach(&txr->pcq_drops); 660 #endif 661 evcnt_detach(&txr->no_desc_avail); 662 evcnt_detach(&txr->tso_tx); 663 664 evcnt_detach(&rxr->rx_packets); 665 evcnt_detach(&rxr->rx_bytes); 666 evcnt_detach(&rxr->rx_copies); 667 evcnt_detach(&rxr->no_mbuf); 668 evcnt_detach(&rxr->rx_discarded); 669 } 670 evcnt_detach(&stats->ipcs); 671 evcnt_detach(&stats->l4cs); 672 evcnt_detach(&stats->ipcs_bad); 673 evcnt_detach(&stats->l4cs_bad); 674 675 /* Packet Reception Stats */ 676 evcnt_detach(&stats->vfgorc); 677 evcnt_detach(&stats->vfgprc); 678 evcnt_detach(&stats->vfmprc); 679 680 /* Packet Transmission Stats */ 681 evcnt_detach(&stats->vfgotc); 682 evcnt_detach(&stats->vfgptc); 683 684 /* Mailbox Stats */ 685 evcnt_detach(&hw->mbx.stats.msgs_tx); 686 evcnt_detach(&hw->mbx.stats.msgs_rx); 687 evcnt_detach(&hw->mbx.stats.acks); 688 evcnt_detach(&hw->mbx.stats.reqs); 689 evcnt_detach(&hw->mbx.stats.rsts); 690 691 ixgbe_free_queues(sc); 692 693 IXGBE_CORE_LOCK_DESTROY(sc); 694 695 return (0); 696 } /* ixv_detach */ 697 698 /************************************************************************ 699 * ixv_init_locked - Init entry point 700 * 701 * Used in two ways: It is used by the stack as an init entry 702 * point in network interface structure. It is also used 703 * by the driver as a hw/sw initialization routine to get 704 * to a consistent state. 705 * 706 * return 0 on success, positive on failure 707 ************************************************************************/ 708 static void 709 ixv_init_locked(struct ixgbe_softc *sc) 710 { 711 struct ifnet *ifp = sc->ifp; 712 device_t dev = sc->dev; 713 struct ixgbe_hw *hw = &sc->hw; 714 struct ix_queue *que; 715 int error = 0; 716 uint32_t mask; 717 int i; 718 719 INIT_DEBUGOUT("ixv_init_locked: begin"); 720 KASSERT(mutex_owned(&sc->core_mtx)); 721 hw->adapter_stopped = FALSE; 722 hw->mac.ops.stop_adapter(hw); 723 callout_stop(&sc->timer); 724 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 725 que->disabled_count = 0; 726 727 sc->max_frame_size = 728 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 729 730 /* reprogram the RAR[0] in case user changed it. */ 731 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 732 733 /* Get the latest mac address, User can use a LAA */ 734 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 735 IXGBE_ETH_LENGTH_OF_ADDRESS); 736 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 737 738 /* Prepare transmit descriptors and buffers */ 739 if (ixgbe_setup_transmit_structures(sc)) { 740 aprint_error_dev(dev, "Could not setup transmit structures\n"); 741 ixv_stop_locked(sc); 742 return; 743 } 744 745 /* Reset VF and renegotiate mailbox API version */ 746 hw->mac.ops.reset_hw(hw); 747 hw->mac.ops.start_hw(hw); 748 error = ixv_negotiate_api(sc); 749 if (error) 750 device_printf(dev, 751 "Mailbox API negotiation failed in init_locked!\n"); 752 753 ixv_initialize_transmit_units(sc); 754 755 /* Setup Multicast table */ 756 ixv_set_rxfilter(sc); 757 758 /* Use fixed buffer size, even for jumbo frames */ 759 sc->rx_mbuf_sz = MCLBYTES; 760 761 /* Prepare receive descriptors and buffers */ 762 error = ixgbe_setup_receive_structures(sc); 763 if (error) { 764 device_printf(dev, 765 "Could not setup receive structures (err = %d)\n", error); 766 ixv_stop_locked(sc); 767 return; 768 } 769 770 /* Configure RX settings */ 771 ixv_initialize_receive_units(sc); 772 773 /* Initialize variable holding task enqueue requests interrupts */ 774 sc->task_requests = 0; 775 776 /* Set up VLAN offload and filter */ 777 ixv_setup_vlan_support(sc); 778 779 /* Set up MSI-X routing */ 780 ixv_configure_ivars(sc); 781 782 /* Set up auto-mask */ 783 mask = (1 << sc->vector); 784 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 785 mask |= (1 << que->msix); 786 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 787 788 /* Set moderation on the Link interrupt */ 789 ixv_eitr_write(sc, sc->vector, IXGBE_LINK_ITR); 790 791 /* Stats init */ 792 ixv_init_stats(sc); 793 794 /* Config/Enable Link */ 795 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 796 if (error) { 797 /* PF is not capable of controlling VF state. Enable the link. */ 798 sc->link_enabled = TRUE; 799 } else if (sc->link_enabled == FALSE) 800 device_printf(dev, "VF is disabled by PF\n"); 801 802 hw->mac.get_link_status = TRUE; 803 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up, 804 FALSE); 805 806 /* Start watchdog */ 807 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 808 atomic_store_relaxed(&sc->timer_pending, 0); 809 810 /* OK to schedule workqueues. */ 811 sc->schedule_wqs_ok = true; 812 813 /* Update saved flags. See ixgbe_ifflags_cb() */ 814 sc->if_flags = ifp->if_flags; 815 sc->ec_capenable = sc->osdep.ec.ec_capenable; 816 817 /* Inform the stack we're ready */ 818 ifp->if_flags |= IFF_RUNNING; 819 820 /* And now turn on interrupts */ 821 ixv_enable_intr(sc); 822 823 return; 824 } /* ixv_init_locked */ 825 826 /************************************************************************ 827 * ixv_enable_queue 828 ************************************************************************/ 829 static inline void 830 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector) 831 { 832 struct ixgbe_hw *hw = &sc->hw; 833 struct ix_queue *que = &sc->queues[vector]; 834 u32 queue = 1UL << vector; 835 u32 mask; 836 837 mutex_enter(&que->dc_mtx); 838 if (que->disabled_count > 0 && --que->disabled_count > 0) 839 goto out; 840 841 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 842 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 843 out: 844 mutex_exit(&que->dc_mtx); 845 } /* ixv_enable_queue */ 846 847 /************************************************************************ 848 * ixv_disable_queue 849 ************************************************************************/ 850 static inline void 851 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector) 852 { 853 struct ixgbe_hw *hw = &sc->hw; 854 struct ix_queue *que = &sc->queues[vector]; 855 u32 queue = 1UL << vector; 856 u32 mask; 857 858 mutex_enter(&que->dc_mtx); 859 if (que->disabled_count++ > 0) 860 goto out; 861 862 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 863 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 864 out: 865 mutex_exit(&que->dc_mtx); 866 } /* ixv_disable_queue */ 867 868 #if 0 869 static inline void 870 ixv_rearm_queues(struct ixgbe_softc *sc, u64 queues) 871 { 872 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 873 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEICS, mask); 874 } /* ixv_rearm_queues */ 875 #endif 876 877 878 /************************************************************************ 879 * ixv_msix_que - MSI-X Queue Interrupt Service routine 880 ************************************************************************/ 881 static int 882 ixv_msix_que(void *arg) 883 { 884 struct ix_queue *que = arg; 885 struct ixgbe_softc *sc = que->sc; 886 struct tx_ring *txr = que->txr; 887 struct rx_ring *rxr = que->rxr; 888 bool more; 889 u32 newitr = 0; 890 891 ixv_disable_queue(sc, que->msix); 892 IXGBE_EVC_ADD(&que->irqs, 1); 893 894 #ifdef __NetBSD__ 895 /* Don't run ixgbe_rxeof in interrupt context */ 896 more = true; 897 #else 898 more = ixgbe_rxeof(que); 899 #endif 900 901 IXGBE_TX_LOCK(txr); 902 ixgbe_txeof(txr); 903 IXGBE_TX_UNLOCK(txr); 904 905 /* Do AIM now? */ 906 907 if (sc->enable_aim == false) 908 goto no_calc; 909 /* 910 * Do Adaptive Interrupt Moderation: 911 * - Write out last calculated setting 912 * - Calculate based on average size over 913 * the last interval. 914 */ 915 if (que->eitr_setting) 916 ixv_eitr_write(sc, que->msix, que->eitr_setting); 917 918 que->eitr_setting = 0; 919 920 /* Idle, do nothing */ 921 if ((txr->bytes == 0) && (rxr->bytes == 0)) 922 goto no_calc; 923 924 if ((txr->bytes) && (txr->packets)) 925 newitr = txr->bytes/txr->packets; 926 if ((rxr->bytes) && (rxr->packets)) 927 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 928 newitr += 24; /* account for hardware frame, crc */ 929 930 /* set an upper boundary */ 931 newitr = uimin(newitr, 3000); 932 933 /* Be nice to the mid range */ 934 if ((newitr > 300) && (newitr < 1200)) 935 newitr = (newitr / 3); 936 else 937 newitr = (newitr / 2); 938 939 /* 940 * When RSC is used, ITR interval must be larger than RSC_DELAY. 941 * Currently, we use 2us for RSC_DELAY. The minimum value is always 942 * greater than 2us on 100M (and 10M?(not documented)), but it's not 943 * on 1G and higher. 944 */ 945 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 946 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 947 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 948 newitr = IXGBE_MIN_RSC_EITR_10G1G; 949 } 950 951 /* save for next interrupt */ 952 que->eitr_setting = newitr; 953 954 /* Reset state */ 955 txr->bytes = 0; 956 txr->packets = 0; 957 rxr->bytes = 0; 958 rxr->packets = 0; 959 960 no_calc: 961 if (more) 962 softint_schedule(que->que_si); 963 else /* Re-enable this interrupt */ 964 ixv_enable_queue(sc, que->msix); 965 966 return 1; 967 } /* ixv_msix_que */ 968 969 /************************************************************************ 970 * ixv_msix_mbx 971 ************************************************************************/ 972 static int 973 ixv_msix_mbx(void *arg) 974 { 975 struct ixgbe_softc *sc = arg; 976 struct ixgbe_hw *hw = &sc->hw; 977 978 IXGBE_EVC_ADD(&sc->admin_irqev, 1); 979 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 980 981 /* Link status change */ 982 hw->mac.get_link_status = TRUE; 983 atomic_or_32(&sc->task_requests, IXGBE_REQUEST_TASK_MBX); 984 ixv_schedule_admin_tasklet(sc); 985 986 return 1; 987 } /* ixv_msix_mbx */ 988 989 static void 990 ixv_eitr_write(struct ixgbe_softc *sc, uint32_t index, uint32_t itr) 991 { 992 993 /* 994 * Newer devices than 82598 have VF function, so this function is 995 * simple. 996 */ 997 itr |= IXGBE_EITR_CNT_WDIS; 998 999 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(index), itr); 1000 } 1001 1002 1003 /************************************************************************ 1004 * ixv_media_status - Media Ioctl callback 1005 * 1006 * Called whenever the user queries the status of 1007 * the interface using ifconfig. 1008 ************************************************************************/ 1009 static void 1010 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1011 { 1012 struct ixgbe_softc *sc = ifp->if_softc; 1013 1014 INIT_DEBUGOUT("ixv_media_status: begin"); 1015 ixv_update_link_status(sc); 1016 1017 ifmr->ifm_status = IFM_AVALID; 1018 ifmr->ifm_active = IFM_ETHER; 1019 1020 if (sc->link_active != LINK_STATE_UP) { 1021 ifmr->ifm_active |= IFM_NONE; 1022 return; 1023 } 1024 1025 ifmr->ifm_status |= IFM_ACTIVE; 1026 1027 switch (sc->link_speed) { 1028 case IXGBE_LINK_SPEED_10GB_FULL: 1029 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1030 break; 1031 case IXGBE_LINK_SPEED_5GB_FULL: 1032 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1033 break; 1034 case IXGBE_LINK_SPEED_2_5GB_FULL: 1035 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1036 break; 1037 case IXGBE_LINK_SPEED_1GB_FULL: 1038 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1039 break; 1040 case IXGBE_LINK_SPEED_100_FULL: 1041 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1042 break; 1043 case IXGBE_LINK_SPEED_10_FULL: 1044 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1045 break; 1046 } 1047 1048 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1049 } /* ixv_media_status */ 1050 1051 /************************************************************************ 1052 * ixv_media_change - Media Ioctl callback 1053 * 1054 * Called when the user changes speed/duplex using 1055 * media/mediopt option with ifconfig. 1056 ************************************************************************/ 1057 static int 1058 ixv_media_change(struct ifnet *ifp) 1059 { 1060 struct ixgbe_softc *sc = ifp->if_softc; 1061 struct ifmedia *ifm = &sc->media; 1062 1063 INIT_DEBUGOUT("ixv_media_change: begin"); 1064 1065 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1066 return (EINVAL); 1067 1068 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1069 case IFM_AUTO: 1070 break; 1071 default: 1072 device_printf(sc->dev, "Only auto media type\n"); 1073 return (EINVAL); 1074 } 1075 1076 return (0); 1077 } /* ixv_media_change */ 1078 1079 static void 1080 ixv_schedule_admin_tasklet(struct ixgbe_softc *sc) 1081 { 1082 if (sc->schedule_wqs_ok) { 1083 if (atomic_cas_uint(&sc->admin_pending, 0, 1) == 0) 1084 workqueue_enqueue(sc->admin_wq, 1085 &sc->admin_wc, NULL); 1086 } 1087 } 1088 1089 /************************************************************************ 1090 * ixv_negotiate_api 1091 * 1092 * Negotiate the Mailbox API with the PF; 1093 * start with the most featured API first. 1094 ************************************************************************/ 1095 static int 1096 ixv_negotiate_api(struct ixgbe_softc *sc) 1097 { 1098 struct ixgbe_hw *hw = &sc->hw; 1099 int mbx_api[] = { ixgbe_mbox_api_15, 1100 ixgbe_mbox_api_13, 1101 ixgbe_mbox_api_12, 1102 ixgbe_mbox_api_11, 1103 ixgbe_mbox_api_10, 1104 ixgbe_mbox_api_unknown }; 1105 int i = 0; 1106 1107 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1108 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) { 1109 if (hw->api_version >= ixgbe_mbox_api_15) 1110 ixgbe_upgrade_mbx_params_vf(hw); 1111 return (0); 1112 } 1113 i++; 1114 } 1115 1116 return (EINVAL); 1117 } /* ixv_negotiate_api */ 1118 1119 1120 /************************************************************************ 1121 * ixv_set_rxfilter - Multicast Update 1122 * 1123 * Called whenever multicast address list is updated. 1124 ************************************************************************/ 1125 static int 1126 ixv_set_rxfilter(struct ixgbe_softc *sc) 1127 { 1128 struct ixgbe_mc_addr *mta; 1129 struct ifnet *ifp = sc->ifp; 1130 struct ixgbe_hw *hw = &sc->hw; 1131 u8 *update_ptr; 1132 int mcnt = 0; 1133 struct ethercom *ec = &sc->osdep.ec; 1134 struct ether_multi *enm; 1135 struct ether_multistep step; 1136 bool overflow = false; 1137 int error, rc = 0; 1138 1139 KASSERT(mutex_owned(&sc->core_mtx)); 1140 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1141 1142 mta = sc->mta; 1143 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1144 1145 /* 1: For PROMISC */ 1146 if (ifp->if_flags & IFF_PROMISC) { 1147 error = hw->mac.ops.update_xcast_mode(hw, 1148 IXGBEVF_XCAST_MODE_PROMISC); 1149 if (error == IXGBE_ERR_NOT_TRUSTED) { 1150 device_printf(sc->dev, 1151 "this interface is not trusted\n"); 1152 error = EPERM; 1153 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1154 device_printf(sc->dev, 1155 "the PF doesn't support promisc mode\n"); 1156 error = EOPNOTSUPP; 1157 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1158 device_printf(sc->dev, 1159 "the PF may not in promisc mode\n"); 1160 error = EINVAL; 1161 } else if (error) { 1162 device_printf(sc->dev, 1163 "failed to set promisc mode. error = %d\n", 1164 error); 1165 error = EIO; 1166 } else 1167 return 0; 1168 rc = error; 1169 } 1170 1171 /* 2: For ALLMULTI or normal */ 1172 ETHER_LOCK(ec); 1173 ETHER_FIRST_MULTI(step, ec, enm); 1174 while (enm != NULL) { 1175 if ((mcnt >= IXGBE_MAX_VF_MC) || 1176 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1177 ETHER_ADDR_LEN) != 0)) { 1178 overflow = true; 1179 break; 1180 } 1181 bcopy(enm->enm_addrlo, 1182 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1183 mcnt++; 1184 ETHER_NEXT_MULTI(step, enm); 1185 } 1186 ETHER_UNLOCK(ec); 1187 1188 /* 3: For ALLMULTI */ 1189 if (overflow) { 1190 error = hw->mac.ops.update_xcast_mode(hw, 1191 IXGBEVF_XCAST_MODE_ALLMULTI); 1192 if (error == IXGBE_ERR_NOT_TRUSTED) { 1193 device_printf(sc->dev, 1194 "this interface is not trusted\n"); 1195 error = EPERM; 1196 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1197 device_printf(sc->dev, 1198 "the PF doesn't support allmulti mode\n"); 1199 error = EOPNOTSUPP; 1200 } else if (error) { 1201 device_printf(sc->dev, 1202 "number of Ethernet multicast addresses " 1203 "exceeds the limit (%d). error = %d\n", 1204 IXGBE_MAX_VF_MC, error); 1205 error = ENOSPC; 1206 } else { 1207 ETHER_LOCK(ec); 1208 ec->ec_flags |= ETHER_F_ALLMULTI; 1209 ETHER_UNLOCK(ec); 1210 return rc; /* Promisc might have failed */ 1211 } 1212 1213 if (rc == 0) 1214 rc = error; 1215 1216 /* Continue to update the multicast table as many as we can */ 1217 } 1218 1219 /* 4: For normal operation */ 1220 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1221 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1222 /* Normal operation */ 1223 ETHER_LOCK(ec); 1224 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1225 ETHER_UNLOCK(ec); 1226 error = 0; 1227 } else if (error) { 1228 device_printf(sc->dev, 1229 "failed to set Ethernet multicast address " 1230 "operation to normal. error = %d\n", error); 1231 } 1232 1233 update_ptr = (u8 *)mta; 1234 error = sc->hw.mac.ops.update_mc_addr_list(&sc->hw, 1235 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1236 if (rc == 0) 1237 rc = error; 1238 1239 return rc; 1240 } /* ixv_set_rxfilter */ 1241 1242 /************************************************************************ 1243 * ixv_mc_array_itr 1244 * 1245 * An iterator function needed by the multicast shared code. 1246 * It feeds the shared code routine the addresses in the 1247 * array of ixv_set_rxfilter() one by one. 1248 ************************************************************************/ 1249 static u8 * 1250 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1251 { 1252 struct ixgbe_mc_addr *mta; 1253 1254 mta = (struct ixgbe_mc_addr *)*update_ptr; 1255 1256 *vmdq = 0; 1257 *update_ptr = (u8*)(mta + 1); 1258 1259 return (mta->addr); 1260 } /* ixv_mc_array_itr */ 1261 1262 /************************************************************************ 1263 * ixv_local_timer - Timer routine 1264 * 1265 * Checks for link status, updates statistics, 1266 * and runs the watchdog check. 1267 ************************************************************************/ 1268 static void 1269 ixv_local_timer(void *arg) 1270 { 1271 struct ixgbe_softc *sc = arg; 1272 1273 if (sc->schedule_wqs_ok) { 1274 if (atomic_cas_uint(&sc->timer_pending, 0, 1) == 0) 1275 workqueue_enqueue(sc->timer_wq, 1276 &sc->timer_wc, NULL); 1277 } 1278 } 1279 1280 static void 1281 ixv_handle_timer(struct work *wk, void *context) 1282 { 1283 struct ixgbe_softc *sc = context; 1284 device_t dev = sc->dev; 1285 struct ix_queue *que = sc->queues; 1286 u64 queues = 0; 1287 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1288 int hung = 0; 1289 int i; 1290 1291 IXGBE_CORE_LOCK(sc); 1292 1293 if (ixv_check_link(sc)) { 1294 ixv_init_locked(sc); 1295 IXGBE_CORE_UNLOCK(sc); 1296 return; 1297 } 1298 1299 /* Stats Update */ 1300 ixv_update_stats(sc); 1301 1302 /* Update some event counters */ 1303 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1304 que = sc->queues; 1305 for (i = 0; i < sc->num_queues; i++, que++) { 1306 struct tx_ring *txr = que->txr; 1307 1308 v0 += txr->q_efbig_tx_dma_setup; 1309 v1 += txr->q_mbuf_defrag_failed; 1310 v2 += txr->q_efbig2_tx_dma_setup; 1311 v3 += txr->q_einval_tx_dma_setup; 1312 v4 += txr->q_other_tx_dma_setup; 1313 v5 += txr->q_eagain_tx_dma_setup; 1314 v6 += txr->q_enomem_tx_dma_setup; 1315 v7 += txr->q_tso_err; 1316 } 1317 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, v0); 1318 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, v1); 1319 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, v2); 1320 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, v3); 1321 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, v4); 1322 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, v5); 1323 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, v6); 1324 IXGBE_EVC_STORE(&sc->tso_err, v7); 1325 1326 /* 1327 * Check the TX queues status 1328 * - mark hung queues so we don't schedule on them 1329 * - watchdog only if all queues show hung 1330 */ 1331 que = sc->queues; 1332 for (i = 0; i < sc->num_queues; i++, que++) { 1333 /* Keep track of queues with work for soft irq */ 1334 if (que->txr->busy) 1335 queues |= ((u64)1 << que->me); 1336 /* 1337 * Each time txeof runs without cleaning, but there 1338 * are uncleaned descriptors it increments busy. If 1339 * we get to the MAX we declare it hung. 1340 */ 1341 if (que->busy == IXGBE_QUEUE_HUNG) { 1342 ++hung; 1343 /* Mark the queue as inactive */ 1344 sc->active_queues &= ~((u64)1 << que->me); 1345 continue; 1346 } else { 1347 /* Check if we've come back from hung */ 1348 if ((sc->active_queues & ((u64)1 << que->me)) == 0) 1349 sc->active_queues |= ((u64)1 << que->me); 1350 } 1351 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1352 device_printf(dev, 1353 "Warning queue %d appears to be hung!\n", i); 1354 que->txr->busy = IXGBE_QUEUE_HUNG; 1355 ++hung; 1356 } 1357 } 1358 1359 /* Only truly watchdog if all queues show hung */ 1360 if (hung == sc->num_queues) 1361 goto watchdog; 1362 #if 0 1363 else if (queues != 0) { /* Force an IRQ on queues with work */ 1364 ixv_rearm_queues(sc, queues); 1365 } 1366 #endif 1367 1368 atomic_store_relaxed(&sc->timer_pending, 0); 1369 IXGBE_CORE_UNLOCK(sc); 1370 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 1371 1372 return; 1373 1374 watchdog: 1375 device_printf(sc->dev, "Watchdog timeout -- resetting\n"); 1376 sc->ifp->if_flags &= ~IFF_RUNNING; 1377 IXGBE_EVC_ADD(&sc->watchdog_events, 1); 1378 ixv_init_locked(sc); 1379 IXGBE_CORE_UNLOCK(sc); 1380 } /* ixv_handle_timer */ 1381 1382 /************************************************************************ 1383 * ixv_update_link_status - Update OS on link state 1384 * 1385 * Note: Only updates the OS on the cached link state. 1386 * The real check of the hardware only happens with 1387 * a link interrupt. 1388 ************************************************************************/ 1389 static void 1390 ixv_update_link_status(struct ixgbe_softc *sc) 1391 { 1392 struct ifnet *ifp = sc->ifp; 1393 device_t dev = sc->dev; 1394 1395 KASSERT(mutex_owned(&sc->core_mtx)); 1396 1397 if (sc->link_up && sc->link_enabled) { 1398 if (sc->link_active != LINK_STATE_UP) { 1399 if (bootverbose) { 1400 const char *bpsmsg; 1401 1402 switch (sc->link_speed) { 1403 case IXGBE_LINK_SPEED_10GB_FULL: 1404 bpsmsg = "10 Gbps"; 1405 break; 1406 case IXGBE_LINK_SPEED_5GB_FULL: 1407 bpsmsg = "5 Gbps"; 1408 break; 1409 case IXGBE_LINK_SPEED_2_5GB_FULL: 1410 bpsmsg = "2.5 Gbps"; 1411 break; 1412 case IXGBE_LINK_SPEED_1GB_FULL: 1413 bpsmsg = "1 Gbps"; 1414 break; 1415 case IXGBE_LINK_SPEED_100_FULL: 1416 bpsmsg = "100 Mbps"; 1417 break; 1418 case IXGBE_LINK_SPEED_10_FULL: 1419 bpsmsg = "10 Mbps"; 1420 break; 1421 default: 1422 bpsmsg = "unknown speed"; 1423 break; 1424 } 1425 device_printf(dev, "Link is up %s %s \n", 1426 bpsmsg, "Full Duplex"); 1427 } 1428 sc->link_active = LINK_STATE_UP; 1429 if_link_state_change(ifp, LINK_STATE_UP); 1430 } 1431 } else { 1432 /* 1433 * Do it when link active changes to DOWN. i.e. 1434 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1435 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1436 */ 1437 if (sc->link_active != LINK_STATE_DOWN) { 1438 if (bootverbose) 1439 device_printf(dev, "Link is Down\n"); 1440 if_link_state_change(ifp, LINK_STATE_DOWN); 1441 sc->link_active = LINK_STATE_DOWN; 1442 } 1443 } 1444 } /* ixv_update_link_status */ 1445 1446 1447 /************************************************************************ 1448 * ixv_stop - Stop the hardware 1449 * 1450 * Disables all traffic on the adapter by issuing a 1451 * global reset on the MAC and deallocates TX/RX buffers. 1452 ************************************************************************/ 1453 static void 1454 ixv_ifstop(struct ifnet *ifp, int disable) 1455 { 1456 struct ixgbe_softc *sc = ifp->if_softc; 1457 1458 IXGBE_CORE_LOCK(sc); 1459 ixv_stop_locked(sc); 1460 IXGBE_CORE_UNLOCK(sc); 1461 1462 workqueue_wait(sc->admin_wq, &sc->admin_wc); 1463 atomic_store_relaxed(&sc->admin_pending, 0); 1464 workqueue_wait(sc->timer_wq, &sc->timer_wc); 1465 atomic_store_relaxed(&sc->timer_pending, 0); 1466 } 1467 1468 static void 1469 ixv_stop_locked(void *arg) 1470 { 1471 struct ifnet *ifp; 1472 struct ixgbe_softc *sc = arg; 1473 struct ixgbe_hw *hw = &sc->hw; 1474 1475 ifp = sc->ifp; 1476 1477 KASSERT(mutex_owned(&sc->core_mtx)); 1478 1479 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1480 ixv_disable_intr(sc); 1481 1482 /* Tell the stack that the interface is no longer active */ 1483 ifp->if_flags &= ~IFF_RUNNING; 1484 1485 hw->mac.ops.reset_hw(hw); 1486 sc->hw.adapter_stopped = FALSE; 1487 hw->mac.ops.stop_adapter(hw); 1488 callout_stop(&sc->timer); 1489 1490 /* Don't schedule workqueues. */ 1491 sc->schedule_wqs_ok = false; 1492 1493 /* reprogram the RAR[0] in case user changed it. */ 1494 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1495 1496 return; 1497 } /* ixv_stop_locked */ 1498 1499 1500 /************************************************************************ 1501 * ixv_allocate_pci_resources 1502 ************************************************************************/ 1503 static int 1504 ixv_allocate_pci_resources(struct ixgbe_softc *sc, 1505 const struct pci_attach_args *pa) 1506 { 1507 pcireg_t memtype, csr; 1508 device_t dev = sc->dev; 1509 bus_addr_t addr; 1510 int flags; 1511 1512 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1513 switch (memtype) { 1514 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1515 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1516 sc->osdep.mem_bus_space_tag = pa->pa_memt; 1517 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1518 memtype, &addr, &sc->osdep.mem_size, &flags) != 0) 1519 goto map_err; 1520 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1521 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1522 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1523 } 1524 if (bus_space_map(sc->osdep.mem_bus_space_tag, addr, 1525 sc->osdep.mem_size, flags, 1526 &sc->osdep.mem_bus_space_handle) != 0) { 1527 map_err: 1528 sc->osdep.mem_size = 0; 1529 aprint_error_dev(dev, "unable to map BAR0\n"); 1530 return ENXIO; 1531 } 1532 /* 1533 * Enable address decoding for memory range in case it's not 1534 * set. 1535 */ 1536 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1537 PCI_COMMAND_STATUS_REG); 1538 csr |= PCI_COMMAND_MEM_ENABLE; 1539 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1540 csr); 1541 break; 1542 default: 1543 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1544 return ENXIO; 1545 } 1546 1547 /* Pick up the tuneable queues */ 1548 sc->num_queues = ixv_num_queues; 1549 1550 return (0); 1551 } /* ixv_allocate_pci_resources */ 1552 1553 static void 1554 ixv_free_deferred_handlers(struct ixgbe_softc *sc) 1555 { 1556 struct ix_queue *que = sc->queues; 1557 struct tx_ring *txr = sc->tx_rings; 1558 int i; 1559 1560 for (i = 0; i < sc->num_queues; i++, que++, txr++) { 1561 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1562 if (txr->txr_si != NULL) 1563 softint_disestablish(txr->txr_si); 1564 } 1565 if (que->que_si != NULL) 1566 softint_disestablish(que->que_si); 1567 } 1568 if (sc->txr_wq != NULL) 1569 workqueue_destroy(sc->txr_wq); 1570 if (sc->txr_wq_enqueued != NULL) 1571 percpu_free(sc->txr_wq_enqueued, sizeof(u_int)); 1572 if (sc->que_wq != NULL) 1573 workqueue_destroy(sc->que_wq); 1574 1575 /* Drain the Mailbox(link) queue */ 1576 if (sc->admin_wq != NULL) { 1577 workqueue_destroy(sc->admin_wq); 1578 sc->admin_wq = NULL; 1579 } 1580 if (sc->timer_wq != NULL) { 1581 workqueue_destroy(sc->timer_wq); 1582 sc->timer_wq = NULL; 1583 } 1584 } /* ixv_free_deferred_handlers */ 1585 1586 /************************************************************************ 1587 * ixv_free_pci_resources 1588 ************************************************************************/ 1589 static void 1590 ixv_free_pci_resources(struct ixgbe_softc *sc) 1591 { 1592 struct ix_queue *que = sc->queues; 1593 int rid; 1594 1595 /* 1596 * Release all msix queue resources: 1597 */ 1598 for (int i = 0; i < sc->num_queues; i++, que++) { 1599 if (que->res != NULL) 1600 pci_intr_disestablish(sc->osdep.pc, 1601 sc->osdep.ihs[i]); 1602 } 1603 1604 1605 /* Clean the Mailbox interrupt last */ 1606 rid = sc->vector; 1607 1608 if (sc->osdep.ihs[rid] != NULL) { 1609 pci_intr_disestablish(sc->osdep.pc, 1610 sc->osdep.ihs[rid]); 1611 sc->osdep.ihs[rid] = NULL; 1612 } 1613 1614 pci_intr_release(sc->osdep.pc, sc->osdep.intrs, 1615 sc->osdep.nintrs); 1616 1617 if (sc->osdep.mem_size != 0) { 1618 bus_space_unmap(sc->osdep.mem_bus_space_tag, 1619 sc->osdep.mem_bus_space_handle, 1620 sc->osdep.mem_size); 1621 } 1622 1623 return; 1624 } /* ixv_free_pci_resources */ 1625 1626 /************************************************************************ 1627 * ixv_setup_interface 1628 * 1629 * Setup networking device structure and register an interface. 1630 ************************************************************************/ 1631 static int 1632 ixv_setup_interface(device_t dev, struct ixgbe_softc *sc) 1633 { 1634 struct ethercom *ec = &sc->osdep.ec; 1635 struct ifnet *ifp; 1636 1637 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1638 1639 ifp = sc->ifp = &ec->ec_if; 1640 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1641 ifp->if_baudrate = IF_Gbps(10); 1642 ifp->if_init = ixv_init; 1643 ifp->if_stop = ixv_ifstop; 1644 ifp->if_softc = sc; 1645 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1646 ifp->if_extflags = IFEF_MPSAFE; 1647 ifp->if_ioctl = ixv_ioctl; 1648 if (sc->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1649 #if 0 1650 ixv_start_locked = ixgbe_legacy_start_locked; 1651 #endif 1652 } else { 1653 ifp->if_transmit = ixgbe_mq_start; 1654 #if 0 1655 ixv_start_locked = ixgbe_mq_start_locked; 1656 #endif 1657 } 1658 ifp->if_start = ixgbe_legacy_start; 1659 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 2); 1660 IFQ_SET_READY(&ifp->if_snd); 1661 1662 if_initialize(ifp); 1663 sc->ipq = if_percpuq_create(&sc->osdep.ec.ec_if); 1664 /* 1665 * We use per TX queue softint, so if_deferred_start_init() isn't 1666 * used. 1667 */ 1668 1669 sc->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1670 1671 /* 1672 * Tell the upper layer(s) we support long frames. 1673 */ 1674 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1675 1676 /* Set capability flags */ 1677 ifp->if_capabilities |= IFCAP_HWCSUM 1678 | IFCAP_TSOv4 1679 | IFCAP_TSOv6; 1680 ifp->if_capenable = 0; 1681 1682 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1683 | ETHERCAP_VLAN_HWTAGGING 1684 | ETHERCAP_VLAN_HWCSUM 1685 | ETHERCAP_JUMBO_MTU 1686 | ETHERCAP_VLAN_MTU; 1687 1688 /* Enable the above capabilities by default */ 1689 ec->ec_capenable = ec->ec_capabilities; 1690 1691 ether_ifattach(ifp, sc->hw.mac.addr); 1692 aprint_normal_dev(dev, "Ethernet address %s\n", 1693 ether_sprintf(sc->hw.mac.addr)); 1694 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1695 1696 /* Don't enable LRO by default */ 1697 #if 0 1698 /* NetBSD doesn't support LRO yet */ 1699 ifp->if_capabilities |= IFCAP_LRO; 1700 #endif 1701 1702 /* 1703 * Specify the media types supported by this adapter and register 1704 * callbacks to update media and link information 1705 */ 1706 ec->ec_ifmedia = &sc->media; 1707 ifmedia_init_with_lock(&sc->media, IFM_IMASK, ixv_media_change, 1708 ixv_media_status, &sc->core_mtx); 1709 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1710 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1711 1712 if_register(ifp); 1713 1714 return 0; 1715 } /* ixv_setup_interface */ 1716 1717 1718 /************************************************************************ 1719 * ixv_initialize_transmit_units - Enable transmit unit. 1720 ************************************************************************/ 1721 static void 1722 ixv_initialize_transmit_units(struct ixgbe_softc *sc) 1723 { 1724 struct tx_ring *txr = sc->tx_rings; 1725 struct ixgbe_hw *hw = &sc->hw; 1726 int i; 1727 1728 for (i = 0; i < sc->num_queues; i++, txr++) { 1729 u64 tdba = txr->txdma.dma_paddr; 1730 u32 txctrl, txdctl; 1731 int j = txr->me; 1732 1733 /* Set WTHRESH to 8, burst writeback */ 1734 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1735 txdctl &= ~IXGBE_TXDCTL_WTHRESH_MASK; 1736 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1737 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1738 1739 /* Set the HW Tx Head and Tail indices */ 1740 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0); 1741 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0); 1742 1743 /* Set Tx Tail register */ 1744 txr->tail = IXGBE_VFTDT(j); 1745 1746 txr->txr_no_space = false; 1747 1748 /* Set Ring parameters */ 1749 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1750 (tdba & 0x00000000ffffffffULL)); 1751 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1752 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1753 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1754 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1755 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1756 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1757 1758 /* Now enable */ 1759 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1760 txdctl |= IXGBE_TXDCTL_ENABLE; 1761 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1762 } 1763 1764 return; 1765 } /* ixv_initialize_transmit_units */ 1766 1767 1768 /************************************************************************ 1769 * ixv_initialize_rss_mapping 1770 ************************************************************************/ 1771 static void 1772 ixv_initialize_rss_mapping(struct ixgbe_softc *sc) 1773 { 1774 struct ixgbe_hw *hw = &sc->hw; 1775 u32 reta = 0, mrqc, rss_key[10]; 1776 int queue_id; 1777 int i, j; 1778 u32 rss_hash_config; 1779 1780 /* force use default RSS key. */ 1781 #ifdef __NetBSD__ 1782 rss_getkey((uint8_t *) &rss_key); 1783 #else 1784 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1785 /* Fetch the configured RSS key */ 1786 rss_getkey((uint8_t *)&rss_key); 1787 } else { 1788 /* set up random bits */ 1789 cprng_fast(&rss_key, sizeof(rss_key)); 1790 } 1791 #endif 1792 1793 /* Now fill out hash function seeds */ 1794 for (i = 0; i < 10; i++) 1795 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1796 1797 /* Set up the redirection table */ 1798 for (i = 0, j = 0; i < 64; i++, j++) { 1799 if (j == sc->num_queues) 1800 j = 0; 1801 1802 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1803 /* 1804 * Fetch the RSS bucket id for the given indirection 1805 * entry. Cap it at the number of configured buckets 1806 * (which is num_queues.) 1807 */ 1808 queue_id = rss_get_indirection_to_bucket(i); 1809 queue_id = queue_id % sc->num_queues; 1810 } else 1811 queue_id = j; 1812 1813 /* 1814 * The low 8 bits are for hash value (n+0); 1815 * The next 8 bits are for hash value (n+1), etc. 1816 */ 1817 reta >>= 8; 1818 reta |= ((uint32_t)queue_id) << 24; 1819 if ((i & 3) == 3) { 1820 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1821 reta = 0; 1822 } 1823 } 1824 1825 /* Perform hash on these packet types */ 1826 if (sc->feat_en & IXGBE_FEATURE_RSS) 1827 rss_hash_config = rss_gethashconfig(); 1828 else { 1829 /* 1830 * Disable UDP - IP fragments aren't currently being handled 1831 * and so we end up with a mix of 2-tuple and 4-tuple 1832 * traffic. 1833 */ 1834 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1835 | RSS_HASHTYPE_RSS_TCP_IPV4 1836 | RSS_HASHTYPE_RSS_IPV6 1837 | RSS_HASHTYPE_RSS_TCP_IPV6; 1838 } 1839 1840 mrqc = IXGBE_MRQC_RSSEN; 1841 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1842 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1843 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1844 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1845 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1846 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1847 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1848 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1849 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1850 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX " 1851 "defined, but not supported\n", __func__); 1852 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1853 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX " 1854 "defined, but not supported\n", __func__); 1855 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1856 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1857 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1858 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1859 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1860 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX " 1861 "defined, but not supported\n", __func__); 1862 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1863 } /* ixv_initialize_rss_mapping */ 1864 1865 1866 /************************************************************************ 1867 * ixv_initialize_receive_units - Setup receive registers and features. 1868 ************************************************************************/ 1869 static void 1870 ixv_initialize_receive_units(struct ixgbe_softc *sc) 1871 { 1872 struct rx_ring *rxr = sc->rx_rings; 1873 struct ixgbe_hw *hw = &sc->hw; 1874 struct ifnet *ifp = sc->ifp; 1875 u32 bufsz, psrtype; 1876 1877 if (ifp->if_mtu > ETHERMTU) 1878 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1879 else 1880 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1881 1882 psrtype = IXGBE_PSRTYPE_TCPHDR 1883 | IXGBE_PSRTYPE_UDPHDR 1884 | IXGBE_PSRTYPE_IPV4HDR 1885 | IXGBE_PSRTYPE_IPV6HDR 1886 | IXGBE_PSRTYPE_L2HDR; 1887 1888 if (sc->num_queues > 1) 1889 psrtype |= 1 << 29; 1890 1891 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1892 1893 /* Tell PF our max_frame size */ 1894 if (ixgbevf_rlpml_set_vf(hw, sc->max_frame_size) != 0) { 1895 device_printf(sc->dev, "There is a problem with the PF " 1896 "setup. It is likely the receive unit for this VF will " 1897 "not function correctly.\n"); 1898 } 1899 1900 for (int i = 0; i < sc->num_queues; i++, rxr++) { 1901 u64 rdba = rxr->rxdma.dma_paddr; 1902 u32 reg, rxdctl; 1903 int j = rxr->me; 1904 1905 /* Disable the queue */ 1906 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1907 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1908 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1909 for (int k = 0; k < 10; k++) { 1910 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1911 IXGBE_RXDCTL_ENABLE) 1912 msec_delay(1); 1913 else 1914 break; 1915 } 1916 IXGBE_WRITE_BARRIER(hw); 1917 /* Setup the Base and Length of the Rx Descriptor Ring */ 1918 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1919 (rdba & 0x00000000ffffffffULL)); 1920 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1921 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1922 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1923 1924 /* Reset the ring indices */ 1925 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1926 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1927 1928 /* Set up the SRRCTL register */ 1929 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1930 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1931 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1932 reg |= bufsz; 1933 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1934 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1935 1936 /* Capture Rx Tail index */ 1937 rxr->tail = IXGBE_VFRDT(rxr->me); 1938 1939 /* Do the queue enabling last */ 1940 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1941 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1942 for (int k = 0; k < 10; k++) { 1943 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1944 IXGBE_RXDCTL_ENABLE) 1945 break; 1946 msec_delay(1); 1947 } 1948 IXGBE_WRITE_BARRIER(hw); 1949 1950 /* Set the Tail Pointer */ 1951 #ifdef DEV_NETMAP 1952 /* 1953 * In netmap mode, we must preserve the buffers made 1954 * available to userspace before the if_init() 1955 * (this is true by default on the TX side, because 1956 * init makes all buffers available to userspace). 1957 * 1958 * netmap_reset() and the device specific routines 1959 * (e.g. ixgbe_setup_receive_rings()) map these 1960 * buffers at the end of the NIC ring, so here we 1961 * must set the RDT (tail) register to make sure 1962 * they are not overwritten. 1963 * 1964 * In this driver the NIC ring starts at RDH = 0, 1965 * RDT points to the last slot available for reception (?), 1966 * so RDT = num_rx_desc - 1 means the whole ring is available. 1967 */ 1968 if ((sc->feat_en & IXGBE_FEATURE_NETMAP) && 1969 (ifp->if_capenable & IFCAP_NETMAP)) { 1970 struct netmap_adapter *na = NA(sc->ifp); 1971 struct netmap_kring *kring = na->rx_rings[i]; 1972 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1973 1974 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1975 } else 1976 #endif /* DEV_NETMAP */ 1977 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1978 sc->num_rx_desc - 1); 1979 } 1980 1981 if (sc->hw.mac.type >= ixgbe_mac_X550_vf) 1982 ixv_initialize_rss_mapping(sc); 1983 } /* ixv_initialize_receive_units */ 1984 1985 /************************************************************************ 1986 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1987 * 1988 * Retrieves the TDH value from the hardware 1989 ************************************************************************/ 1990 static int 1991 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1992 { 1993 struct sysctlnode node = *rnode; 1994 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1995 uint32_t val; 1996 1997 if (!txr) 1998 return (0); 1999 2000 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDH(txr->me)); 2001 node.sysctl_data = &val; 2002 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2003 } /* ixv_sysctl_tdh_handler */ 2004 2005 /************************************************************************ 2006 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2007 * 2008 * Retrieves the TDT value from the hardware 2009 ************************************************************************/ 2010 static int 2011 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 2012 { 2013 struct sysctlnode node = *rnode; 2014 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2015 uint32_t val; 2016 2017 if (!txr) 2018 return (0); 2019 2020 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDT(txr->me)); 2021 node.sysctl_data = &val; 2022 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2023 } /* ixv_sysctl_tdt_handler */ 2024 2025 /************************************************************************ 2026 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2027 * handler function 2028 * 2029 * Retrieves the next_to_check value 2030 ************************************************************************/ 2031 static int 2032 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2033 { 2034 struct sysctlnode node = *rnode; 2035 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2036 uint32_t val; 2037 2038 if (!rxr) 2039 return (0); 2040 2041 val = rxr->next_to_check; 2042 node.sysctl_data = &val; 2043 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2044 } /* ixv_sysctl_next_to_check_handler */ 2045 2046 /************************************************************************ 2047 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2048 * handler function 2049 * 2050 * Retrieves the next_to_refresh value 2051 ************************************************************************/ 2052 static int 2053 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2054 { 2055 struct sysctlnode node = *rnode; 2056 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2057 struct ixgbe_softc *sc; 2058 uint32_t val; 2059 2060 if (!rxr) 2061 return (0); 2062 2063 sc = rxr->sc; 2064 if (ixgbe_fw_recovery_mode_swflag(sc)) 2065 return (EPERM); 2066 2067 val = rxr->next_to_refresh; 2068 node.sysctl_data = &val; 2069 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2070 } /* ixv_sysctl_next_to_refresh_handler */ 2071 2072 /************************************************************************ 2073 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2074 * 2075 * Retrieves the RDH value from the hardware 2076 ************************************************************************/ 2077 static int 2078 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2079 { 2080 struct sysctlnode node = *rnode; 2081 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2082 uint32_t val; 2083 2084 if (!rxr) 2085 return (0); 2086 2087 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDH(rxr->me)); 2088 node.sysctl_data = &val; 2089 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2090 } /* ixv_sysctl_rdh_handler */ 2091 2092 /************************************************************************ 2093 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2094 * 2095 * Retrieves the RDT value from the hardware 2096 ************************************************************************/ 2097 static int 2098 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2099 { 2100 struct sysctlnode node = *rnode; 2101 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2102 uint32_t val; 2103 2104 if (!rxr) 2105 return (0); 2106 2107 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDT(rxr->me)); 2108 node.sysctl_data = &val; 2109 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2110 } /* ixv_sysctl_rdt_handler */ 2111 2112 static void 2113 ixv_setup_vlan_tagging(struct ixgbe_softc *sc) 2114 { 2115 struct ethercom *ec = &sc->osdep.ec; 2116 struct ixgbe_hw *hw = &sc->hw; 2117 struct rx_ring *rxr; 2118 u32 ctrl; 2119 int i; 2120 bool hwtagging; 2121 2122 /* Enable HW tagging only if any vlan is attached */ 2123 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2124 && VLAN_ATTACHED(ec); 2125 2126 /* Enable the queues */ 2127 for (i = 0; i < sc->num_queues; i++) { 2128 rxr = &sc->rx_rings[i]; 2129 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2130 if (hwtagging) 2131 ctrl |= IXGBE_RXDCTL_VME; 2132 else 2133 ctrl &= ~IXGBE_RXDCTL_VME; 2134 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2135 /* 2136 * Let Rx path know that it needs to store VLAN tag 2137 * as part of extra mbuf info. 2138 */ 2139 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2140 } 2141 } /* ixv_setup_vlan_tagging */ 2142 2143 /************************************************************************ 2144 * ixv_setup_vlan_support 2145 ************************************************************************/ 2146 static int 2147 ixv_setup_vlan_support(struct ixgbe_softc *sc) 2148 { 2149 struct ethercom *ec = &sc->osdep.ec; 2150 struct ixgbe_hw *hw = &sc->hw; 2151 u32 vid, vfta, retry; 2152 struct vlanid_list *vlanidp; 2153 int rv, error = 0; 2154 2155 /* 2156 * This function is called from both if_init and ifflags_cb() 2157 * on NetBSD. 2158 */ 2159 2160 /* 2161 * Part 1: 2162 * Setup VLAN HW tagging 2163 */ 2164 ixv_setup_vlan_tagging(sc); 2165 2166 if (!VLAN_ATTACHED(ec)) 2167 return 0; 2168 2169 /* 2170 * Part 2: 2171 * Setup VLAN HW filter 2172 */ 2173 /* Cleanup shadow_vfta */ 2174 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2175 sc->shadow_vfta[i] = 0; 2176 /* Generate shadow_vfta from ec_vids */ 2177 ETHER_LOCK(ec); 2178 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2179 uint32_t idx; 2180 2181 idx = vlanidp->vid / 32; 2182 KASSERT(idx < IXGBE_VFTA_SIZE); 2183 sc->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2184 } 2185 ETHER_UNLOCK(ec); 2186 2187 /* 2188 * A soft reset zero's out the VFTA, so 2189 * we need to repopulate it now. 2190 */ 2191 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2192 if (sc->shadow_vfta[i] == 0) 2193 continue; 2194 vfta = sc->shadow_vfta[i]; 2195 /* 2196 * Reconstruct the vlan id's 2197 * based on the bits set in each 2198 * of the array ints. 2199 */ 2200 for (int j = 0; j < 32; j++) { 2201 retry = 0; 2202 if ((vfta & ((u32)1 << j)) == 0) 2203 continue; 2204 vid = (i * 32) + j; 2205 2206 /* Call the shared code mailbox routine */ 2207 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2208 FALSE)) != 0) { 2209 if (++retry > 5) { 2210 device_printf(sc->dev, 2211 "%s: max retry exceeded\n", 2212 __func__); 2213 break; 2214 } 2215 } 2216 if (rv != 0) { 2217 device_printf(sc->dev, 2218 "failed to set vlan %d\n", vid); 2219 error = EACCES; 2220 } 2221 } 2222 } 2223 return error; 2224 } /* ixv_setup_vlan_support */ 2225 2226 static int 2227 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2228 { 2229 struct ifnet *ifp = &ec->ec_if; 2230 struct ixgbe_softc *sc = ifp->if_softc; 2231 int rv; 2232 2233 if (set) 2234 rv = ixv_register_vlan(sc, vid); 2235 else 2236 rv = ixv_unregister_vlan(sc, vid); 2237 2238 if (rv != 0) 2239 return rv; 2240 2241 /* 2242 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2243 * or 0 to 1. 2244 */ 2245 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2246 ixv_setup_vlan_tagging(sc); 2247 2248 return rv; 2249 } 2250 2251 /************************************************************************ 2252 * ixv_register_vlan 2253 * 2254 * Run via a vlan config EVENT, it enables us to use the 2255 * HW Filter table since we can get the vlan id. This just 2256 * creates the entry in the soft version of the VFTA, init 2257 * will repopulate the real table. 2258 ************************************************************************/ 2259 static int 2260 ixv_register_vlan(struct ixgbe_softc *sc, u16 vtag) 2261 { 2262 struct ixgbe_hw *hw = &sc->hw; 2263 u16 index, bit; 2264 int error; 2265 2266 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2267 return EINVAL; 2268 IXGBE_CORE_LOCK(sc); 2269 index = (vtag >> 5) & 0x7F; 2270 bit = vtag & 0x1F; 2271 sc->shadow_vfta[index] |= ((u32)1 << bit); 2272 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2273 IXGBE_CORE_UNLOCK(sc); 2274 2275 if (error != 0) { 2276 device_printf(sc->dev, "failed to register vlan %hu\n", vtag); 2277 error = EACCES; 2278 } 2279 return error; 2280 } /* ixv_register_vlan */ 2281 2282 /************************************************************************ 2283 * ixv_unregister_vlan 2284 * 2285 * Run via a vlan unconfig EVENT, remove our entry 2286 * in the soft vfta. 2287 ************************************************************************/ 2288 static int 2289 ixv_unregister_vlan(struct ixgbe_softc *sc, u16 vtag) 2290 { 2291 struct ixgbe_hw *hw = &sc->hw; 2292 u16 index, bit; 2293 int error; 2294 2295 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2296 return EINVAL; 2297 2298 IXGBE_CORE_LOCK(sc); 2299 index = (vtag >> 5) & 0x7F; 2300 bit = vtag & 0x1F; 2301 sc->shadow_vfta[index] &= ~((u32)1 << bit); 2302 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2303 IXGBE_CORE_UNLOCK(sc); 2304 2305 if (error != 0) { 2306 device_printf(sc->dev, "failed to unregister vlan %hu\n", 2307 vtag); 2308 error = EIO; 2309 } 2310 return error; 2311 } /* ixv_unregister_vlan */ 2312 2313 /************************************************************************ 2314 * ixv_enable_intr 2315 ************************************************************************/ 2316 static void 2317 ixv_enable_intr(struct ixgbe_softc *sc) 2318 { 2319 struct ixgbe_hw *hw = &sc->hw; 2320 struct ix_queue *que = sc->queues; 2321 u32 mask; 2322 int i; 2323 2324 /* For VTEIAC */ 2325 mask = (1 << sc->vector); 2326 for (i = 0; i < sc->num_queues; i++, que++) 2327 mask |= (1 << que->msix); 2328 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2329 2330 /* For VTEIMS */ 2331 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 2332 que = sc->queues; 2333 for (i = 0; i < sc->num_queues; i++, que++) 2334 ixv_enable_queue(sc, que->msix); 2335 2336 IXGBE_WRITE_FLUSH(hw); 2337 } /* ixv_enable_intr */ 2338 2339 /************************************************************************ 2340 * ixv_disable_intr 2341 ************************************************************************/ 2342 static void 2343 ixv_disable_intr(struct ixgbe_softc *sc) 2344 { 2345 struct ix_queue *que = sc->queues; 2346 2347 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0); 2348 2349 /* disable interrupts other than queues */ 2350 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, sc->vector); 2351 2352 for (int i = 0; i < sc->num_queues; i++, que++) 2353 ixv_disable_queue(sc, que->msix); 2354 2355 IXGBE_WRITE_FLUSH(&sc->hw); 2356 } /* ixv_disable_intr */ 2357 2358 /************************************************************************ 2359 * ixv_set_ivar 2360 * 2361 * Setup the correct IVAR register for a particular MSI-X interrupt 2362 * - entry is the register array entry 2363 * - vector is the MSI-X vector for this queue 2364 * - type is RX/TX/MISC 2365 ************************************************************************/ 2366 static void 2367 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type) 2368 { 2369 struct ixgbe_hw *hw = &sc->hw; 2370 u32 ivar, index; 2371 2372 vector |= IXGBE_IVAR_ALLOC_VAL; 2373 2374 if (type == -1) { /* MISC IVAR */ 2375 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2376 ivar &= ~0xFF; 2377 ivar |= vector; 2378 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2379 } else { /* RX/TX IVARS */ 2380 index = (16 * (entry & 1)) + (8 * type); 2381 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2382 ivar &= ~(0xffUL << index); 2383 ivar |= ((u32)vector << index); 2384 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2385 } 2386 } /* ixv_set_ivar */ 2387 2388 /************************************************************************ 2389 * ixv_configure_ivars 2390 ************************************************************************/ 2391 static void 2392 ixv_configure_ivars(struct ixgbe_softc *sc) 2393 { 2394 struct ix_queue *que = sc->queues; 2395 2396 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2397 2398 for (int i = 0; i < sc->num_queues; i++, que++) { 2399 /* First the RX queue entry */ 2400 ixv_set_ivar(sc, i, que->msix, 0); 2401 /* ... and the TX */ 2402 ixv_set_ivar(sc, i, que->msix, 1); 2403 /* Set an initial value in EITR */ 2404 ixv_eitr_write(sc, que->msix, IXGBE_EITR_DEFAULT); 2405 } 2406 2407 /* For the mailbox interrupt */ 2408 ixv_set_ivar(sc, 1, sc->vector, -1); 2409 } /* ixv_configure_ivars */ 2410 2411 2412 /************************************************************************ 2413 * ixv_init_stats 2414 * 2415 * The VF stats registers never have a truly virgin 2416 * starting point, so this routine save initial vaules to 2417 * last_<REGNAME>. 2418 ************************************************************************/ 2419 static void 2420 ixv_init_stats(struct ixgbe_softc *sc) 2421 { 2422 struct ixgbe_hw *hw = &sc->hw; 2423 2424 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2425 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2426 sc->stats.vf.last_vfgorc |= 2427 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2428 2429 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2430 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2431 sc->stats.vf.last_vfgotc |= 2432 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2433 2434 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2435 } /* ixv_init_stats */ 2436 2437 #define UPDATE_STAT_32(reg, last, count) \ 2438 { \ 2439 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2440 IXGBE_EVC_ADD(&count, current - (last)); \ 2441 (last) = current; \ 2442 } 2443 2444 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2445 { \ 2446 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2447 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2448 u64 current = ((cur_msb << 32) | cur_lsb); \ 2449 if (current < (last)) \ 2450 IXGBE_EVC_ADD(&count, current + __BIT(36) - (last)); \ 2451 else \ 2452 IXGBE_EVC_ADD(&count, current - (last)); \ 2453 (last) = current; \ 2454 } 2455 2456 /************************************************************************ 2457 * ixv_update_stats - Update the board statistics counters. 2458 ************************************************************************/ 2459 void 2460 ixv_update_stats(struct ixgbe_softc *sc) 2461 { 2462 struct ixgbe_hw *hw = &sc->hw; 2463 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2464 2465 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2466 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2467 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2468 stats->vfgorc); 2469 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2470 stats->vfgotc); 2471 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2472 2473 /* VF doesn't count errors by hardware */ 2474 2475 } /* ixv_update_stats */ 2476 2477 /************************************************************************ 2478 * ixv_sysctl_interrupt_rate_handler 2479 ************************************************************************/ 2480 static int 2481 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2482 { 2483 struct sysctlnode node = *rnode; 2484 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2485 struct ixgbe_softc *sc = que->sc; 2486 uint32_t reg, usec, rate; 2487 int error; 2488 2489 if (que == NULL) 2490 return 0; 2491 reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_VTEITR(que->msix)); 2492 usec = ((reg & 0x0FF8) >> 3); 2493 if (usec > 0) 2494 rate = 500000 / usec; 2495 else 2496 rate = 0; 2497 node.sysctl_data = &rate; 2498 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2499 if (error || newp == NULL) 2500 return error; 2501 reg &= ~0xfff; /* default, no limitation */ 2502 if (rate > 0 && rate < 500000) { 2503 if (rate < 1000) 2504 rate = 1000; 2505 reg |= ((4000000 / rate) & 0xff8); 2506 /* 2507 * When RSC is used, ITR interval must be larger than 2508 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2509 * The minimum value is always greater than 2us on 100M 2510 * (and 10M?(not documented)), but it's not on 1G and higher. 2511 */ 2512 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 2513 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2514 if ((sc->num_queues > 1) 2515 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2516 return EINVAL; 2517 } 2518 sc->max_interrupt_rate = rate; 2519 } else 2520 sc->max_interrupt_rate = 0; 2521 ixv_eitr_write(sc, que->msix, reg); 2522 2523 return (0); 2524 } /* ixv_sysctl_interrupt_rate_handler */ 2525 2526 const struct sysctlnode * 2527 ixv_sysctl_instance(struct ixgbe_softc *sc) 2528 { 2529 const char *dvname; 2530 struct sysctllog **log; 2531 int rc; 2532 const struct sysctlnode *rnode; 2533 2534 log = &sc->sysctllog; 2535 dvname = device_xname(sc->dev); 2536 2537 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2538 0, CTLTYPE_NODE, dvname, 2539 SYSCTL_DESCR("ixv information and settings"), 2540 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2541 goto err; 2542 2543 return rnode; 2544 err: 2545 device_printf(sc->dev, 2546 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2547 return NULL; 2548 } 2549 2550 static void 2551 ixv_add_device_sysctls(struct ixgbe_softc *sc) 2552 { 2553 struct sysctllog **log; 2554 const struct sysctlnode *rnode, *cnode; 2555 device_t dev; 2556 2557 dev = sc->dev; 2558 log = &sc->sysctllog; 2559 2560 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2561 aprint_error_dev(dev, "could not create sysctl root\n"); 2562 return; 2563 } 2564 2565 if (sysctl_createv(log, 0, &rnode, &cnode, 2566 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2567 SYSCTL_DESCR("Debug Info"), 2568 ixv_sysctl_debug, 0, (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2569 aprint_error_dev(dev, "could not create sysctl\n"); 2570 2571 if (sysctl_createv(log, 0, &rnode, &cnode, 2572 CTLFLAG_READWRITE, CTLTYPE_INT, 2573 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2574 ixv_sysctl_rx_copy_len, 0, 2575 (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2576 aprint_error_dev(dev, "could not create sysctl\n"); 2577 2578 if (sysctl_createv(log, 0, &rnode, &cnode, 2579 CTLFLAG_READONLY, CTLTYPE_INT, 2580 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 2581 NULL, 0, &sc->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2582 aprint_error_dev(dev, "could not create sysctl\n"); 2583 2584 if (sysctl_createv(log, 0, &rnode, &cnode, 2585 CTLFLAG_READONLY, CTLTYPE_INT, 2586 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 2587 NULL, 0, &sc->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2588 aprint_error_dev(dev, "could not create sysctl\n"); 2589 2590 if (sysctl_createv(log, 0, &rnode, &cnode, 2591 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 2592 SYSCTL_DESCR("max number of RX packets to process"), 2593 ixv_sysctl_rx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2594 CTL_EOL) != 0) 2595 aprint_error_dev(dev, "could not create sysctl\n"); 2596 2597 if (sysctl_createv(log, 0, &rnode, &cnode, 2598 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 2599 SYSCTL_DESCR("max number of TX packets to process"), 2600 ixv_sysctl_tx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2601 CTL_EOL) != 0) 2602 aprint_error_dev(dev, "could not create sysctl\n"); 2603 2604 if (sysctl_createv(log, 0, &rnode, &cnode, 2605 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2606 SYSCTL_DESCR("Interrupt Moderation"), 2607 NULL, 0, &sc->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2608 aprint_error_dev(dev, "could not create sysctl\n"); 2609 2610 if (sysctl_createv(log, 0, &rnode, &cnode, 2611 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2612 SYSCTL_DESCR("Use workqueue for packet processing"), 2613 NULL, 0, &sc->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2614 != 0) 2615 aprint_error_dev(dev, "could not create sysctl\n"); 2616 } 2617 2618 /************************************************************************ 2619 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2620 ************************************************************************/ 2621 static void 2622 ixv_add_stats_sysctls(struct ixgbe_softc *sc) 2623 { 2624 device_t dev = sc->dev; 2625 struct tx_ring *txr = sc->tx_rings; 2626 struct rx_ring *rxr = sc->rx_rings; 2627 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2628 struct ixgbe_hw *hw = &sc->hw; 2629 const struct sysctlnode *rnode, *cnode; 2630 struct sysctllog **log = &sc->sysctllog; 2631 const char *xname = device_xname(dev); 2632 2633 /* Driver Statistics */ 2634 evcnt_attach_dynamic(&sc->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2635 NULL, xname, "Driver tx dma soft fail EFBIG"); 2636 evcnt_attach_dynamic(&sc->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2637 NULL, xname, "m_defrag() failed"); 2638 evcnt_attach_dynamic(&sc->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2639 NULL, xname, "Driver tx dma hard fail EFBIG"); 2640 evcnt_attach_dynamic(&sc->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2641 NULL, xname, "Driver tx dma hard fail EINVAL"); 2642 evcnt_attach_dynamic(&sc->other_tx_dma_setup, EVCNT_TYPE_MISC, 2643 NULL, xname, "Driver tx dma hard fail other"); 2644 evcnt_attach_dynamic(&sc->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2645 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2646 evcnt_attach_dynamic(&sc->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2647 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2648 evcnt_attach_dynamic(&sc->watchdog_events, EVCNT_TYPE_MISC, 2649 NULL, xname, "Watchdog timeouts"); 2650 evcnt_attach_dynamic(&sc->tso_err, EVCNT_TYPE_MISC, 2651 NULL, xname, "TSO errors"); 2652 evcnt_attach_dynamic(&sc->admin_irqev, EVCNT_TYPE_INTR, 2653 NULL, xname, "Admin MSI-X IRQ Handled"); 2654 evcnt_attach_dynamic(&sc->link_workev, EVCNT_TYPE_INTR, 2655 NULL, xname, "Admin event"); 2656 2657 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2658 #ifdef LRO 2659 struct lro_ctrl *lro = &rxr->lro; 2660 #endif 2661 2662 snprintf(sc->queues[i].evnamebuf, 2663 sizeof(sc->queues[i].evnamebuf), "%s q%d", xname, i); 2664 snprintf(sc->queues[i].namebuf, 2665 sizeof(sc->queues[i].namebuf), "q%d", i); 2666 2667 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2668 aprint_error_dev(dev, 2669 "could not create sysctl root\n"); 2670 break; 2671 } 2672 2673 if (sysctl_createv(log, 0, &rnode, &rnode, 2674 0, CTLTYPE_NODE, 2675 sc->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2676 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2677 break; 2678 2679 if (sysctl_createv(log, 0, &rnode, &cnode, 2680 CTLFLAG_READWRITE, CTLTYPE_INT, 2681 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2682 ixv_sysctl_interrupt_rate_handler, 0, 2683 (void *)&sc->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2684 break; 2685 2686 if (sysctl_createv(log, 0, &rnode, &cnode, 2687 CTLFLAG_READONLY, CTLTYPE_INT, 2688 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2689 ixv_sysctl_tdh_handler, 0, (void *)txr, 2690 0, CTL_CREATE, CTL_EOL) != 0) 2691 break; 2692 2693 if (sysctl_createv(log, 0, &rnode, &cnode, 2694 CTLFLAG_READONLY, CTLTYPE_INT, 2695 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2696 ixv_sysctl_tdt_handler, 0, (void *)txr, 2697 0, CTL_CREATE, CTL_EOL) != 0) 2698 break; 2699 2700 if (sysctl_createv(log, 0, &rnode, &cnode, 2701 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2702 SYSCTL_DESCR("Receive Descriptor next to check"), 2703 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2704 CTL_CREATE, CTL_EOL) != 0) 2705 break; 2706 2707 if (sysctl_createv(log, 0, &rnode, &cnode, 2708 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2709 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2710 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2711 CTL_CREATE, CTL_EOL) != 0) 2712 break; 2713 2714 if (sysctl_createv(log, 0, &rnode, &cnode, 2715 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2716 SYSCTL_DESCR("Receive Descriptor Head"), 2717 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2718 CTL_CREATE, CTL_EOL) != 0) 2719 break; 2720 2721 if (sysctl_createv(log, 0, &rnode, &cnode, 2722 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2723 SYSCTL_DESCR("Receive Descriptor Tail"), 2724 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2725 CTL_CREATE, CTL_EOL) != 0) 2726 break; 2727 2728 evcnt_attach_dynamic(&sc->queues[i].irqs, EVCNT_TYPE_INTR, 2729 NULL, sc->queues[i].evnamebuf, "IRQs on queue"); 2730 evcnt_attach_dynamic(&sc->queues[i].handleq, 2731 EVCNT_TYPE_MISC, NULL, sc->queues[i].evnamebuf, 2732 "Handled queue in softint"); 2733 evcnt_attach_dynamic(&sc->queues[i].req, EVCNT_TYPE_MISC, 2734 NULL, sc->queues[i].evnamebuf, "Requeued in softint"); 2735 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2736 NULL, sc->queues[i].evnamebuf, 2737 "Queue Packets Transmitted"); 2738 #ifndef IXGBE_LEGACY_TX 2739 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2740 NULL, sc->queues[i].evnamebuf, 2741 "Packets dropped in pcq"); 2742 #endif 2743 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2744 NULL, sc->queues[i].evnamebuf, 2745 "TX Queue No Descriptor Available"); 2746 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2747 NULL, sc->queues[i].evnamebuf, "TSO"); 2748 2749 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2750 NULL, sc->queues[i].evnamebuf, 2751 "Queue Bytes Received"); 2752 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2753 NULL, sc->queues[i].evnamebuf, 2754 "Queue Packets Received"); 2755 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2756 NULL, sc->queues[i].evnamebuf, "Rx no mbuf"); 2757 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2758 NULL, sc->queues[i].evnamebuf, "Rx discarded"); 2759 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2760 NULL, sc->queues[i].evnamebuf, "Copied RX Frames"); 2761 #ifdef LRO 2762 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2763 CTLFLAG_RD, &lro->lro_queued, 0, 2764 "LRO Queued"); 2765 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2766 CTLFLAG_RD, &lro->lro_flushed, 0, 2767 "LRO Flushed"); 2768 #endif /* LRO */ 2769 } 2770 2771 /* MAC stats get their own sub node */ 2772 2773 snprintf(stats->namebuf, 2774 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2775 2776 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2777 stats->namebuf, "rx csum offload - IP"); 2778 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2779 stats->namebuf, "rx csum offload - L4"); 2780 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2781 stats->namebuf, "rx csum offload - IP bad"); 2782 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2783 stats->namebuf, "rx csum offload - L4 bad"); 2784 2785 /* Packet Reception Stats */ 2786 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2787 xname, "Good Packets Received"); 2788 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2789 xname, "Good Octets Received"); 2790 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2791 xname, "Multicast Packets Received"); 2792 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2793 xname, "Good Packets Transmitted"); 2794 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2795 xname, "Good Octets Transmitted"); 2796 2797 /* Mailbox Stats */ 2798 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2799 xname, "message TXs"); 2800 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2801 xname, "message RXs"); 2802 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2803 xname, "ACKs"); 2804 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2805 xname, "REQs"); 2806 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2807 xname, "RSTs"); 2808 2809 } /* ixv_add_stats_sysctls */ 2810 2811 static void 2812 ixv_clear_evcnt(struct ixgbe_softc *sc) 2813 { 2814 struct tx_ring *txr = sc->tx_rings; 2815 struct rx_ring *rxr = sc->rx_rings; 2816 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2817 struct ixgbe_hw *hw = &sc->hw; 2818 int i; 2819 2820 /* Driver Statistics */ 2821 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, 0); 2822 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, 0); 2823 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, 0); 2824 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, 0); 2825 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, 0); 2826 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, 0); 2827 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, 0); 2828 IXGBE_EVC_STORE(&sc->watchdog_events, 0); 2829 IXGBE_EVC_STORE(&sc->tso_err, 0); 2830 IXGBE_EVC_STORE(&sc->admin_irqev, 0); 2831 IXGBE_EVC_STORE(&sc->link_workev, 0); 2832 2833 for (i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2834 IXGBE_EVC_STORE(&sc->queues[i].irqs, 0); 2835 IXGBE_EVC_STORE(&sc->queues[i].handleq, 0); 2836 IXGBE_EVC_STORE(&sc->queues[i].req, 0); 2837 IXGBE_EVC_STORE(&txr->total_packets, 0); 2838 #ifndef IXGBE_LEGACY_TX 2839 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2840 #endif 2841 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2842 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2843 txr->q_efbig_tx_dma_setup = 0; 2844 txr->q_mbuf_defrag_failed = 0; 2845 txr->q_efbig2_tx_dma_setup = 0; 2846 txr->q_einval_tx_dma_setup = 0; 2847 txr->q_other_tx_dma_setup = 0; 2848 txr->q_eagain_tx_dma_setup = 0; 2849 txr->q_enomem_tx_dma_setup = 0; 2850 txr->q_tso_err = 0; 2851 2852 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2853 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2854 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2855 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2856 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2857 } 2858 2859 /* MAC stats get their own sub node */ 2860 2861 IXGBE_EVC_STORE(&stats->ipcs, 0); 2862 IXGBE_EVC_STORE(&stats->l4cs, 0); 2863 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2864 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2865 2866 /* 2867 * Packet Reception Stats. 2868 * Call ixv_init_stats() to save last VF counters' values. 2869 */ 2870 ixv_init_stats(sc); 2871 IXGBE_EVC_STORE(&stats->vfgprc, 0); 2872 IXGBE_EVC_STORE(&stats->vfgorc, 0); 2873 IXGBE_EVC_STORE(&stats->vfmprc, 0); 2874 IXGBE_EVC_STORE(&stats->vfgptc, 0); 2875 IXGBE_EVC_STORE(&stats->vfgotc, 0); 2876 2877 /* Mailbox Stats */ 2878 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_tx, 0); 2879 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_rx, 0); 2880 IXGBE_EVC_STORE(&hw->mbx.stats.acks, 0); 2881 IXGBE_EVC_STORE(&hw->mbx.stats.reqs, 0); 2882 IXGBE_EVC_STORE(&hw->mbx.stats.rsts, 0); 2883 2884 } /* ixv_clear_evcnt */ 2885 2886 #define PRINTQS(sc, regname) \ 2887 do { \ 2888 struct ixgbe_hw *_hw = &(sc)->hw; \ 2889 int _i; \ 2890 \ 2891 printf("%s: %s", device_xname((sc)->dev), #regname); \ 2892 for (_i = 0; _i < (sc)->num_queues; _i++) { \ 2893 printf((_i == 0) ? "\t" : " "); \ 2894 printf("%08x", IXGBE_READ_REG(_hw, \ 2895 IXGBE_##regname(_i))); \ 2896 } \ 2897 printf("\n"); \ 2898 } while (0) 2899 2900 /************************************************************************ 2901 * ixv_print_debug_info 2902 * 2903 * Provides a way to take a look at important statistics 2904 * maintained by the driver and hardware. 2905 ************************************************************************/ 2906 static void 2907 ixv_print_debug_info(struct ixgbe_softc *sc) 2908 { 2909 device_t dev = sc->dev; 2910 struct ixgbe_hw *hw = &sc->hw; 2911 int i; 2912 2913 device_printf(dev, "queue:"); 2914 for (i = 0; i < sc->num_queues; i++) { 2915 printf((i == 0) ? "\t" : " "); 2916 printf("%8d", i); 2917 } 2918 printf("\n"); 2919 PRINTQS(sc, VFRDBAL); 2920 PRINTQS(sc, VFRDBAH); 2921 PRINTQS(sc, VFRDLEN); 2922 PRINTQS(sc, VFSRRCTL); 2923 PRINTQS(sc, VFRDH); 2924 PRINTQS(sc, VFRDT); 2925 PRINTQS(sc, VFRXDCTL); 2926 2927 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIMS)); 2928 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAM)); 2929 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAC)); 2930 } /* ixv_print_debug_info */ 2931 2932 /************************************************************************ 2933 * ixv_sysctl_debug 2934 ************************************************************************/ 2935 static int 2936 ixv_sysctl_debug(SYSCTLFN_ARGS) 2937 { 2938 struct sysctlnode node = *rnode; 2939 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2940 int error, result = 0; 2941 2942 node.sysctl_data = &result; 2943 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2944 2945 if (error || newp == NULL) 2946 return error; 2947 2948 if (result == 1) 2949 ixv_print_debug_info(sc); 2950 2951 return 0; 2952 } /* ixv_sysctl_debug */ 2953 2954 /************************************************************************ 2955 * ixv_sysctl_rx_copy_len 2956 ************************************************************************/ 2957 static int 2958 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2959 { 2960 struct sysctlnode node = *rnode; 2961 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2962 int error; 2963 int result = sc->rx_copy_len; 2964 2965 node.sysctl_data = &result; 2966 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2967 2968 if (error || newp == NULL) 2969 return error; 2970 2971 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2972 return EINVAL; 2973 2974 sc->rx_copy_len = result; 2975 2976 return 0; 2977 } /* ixv_sysctl_rx_copy_len */ 2978 2979 /************************************************************************ 2980 * ixv_sysctl_tx_process_limit 2981 ************************************************************************/ 2982 static int 2983 ixv_sysctl_tx_process_limit(SYSCTLFN_ARGS) 2984 { 2985 struct sysctlnode node = *rnode; 2986 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2987 int error; 2988 int result = sc->tx_process_limit; 2989 2990 node.sysctl_data = &result; 2991 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2992 2993 if (error || newp == NULL) 2994 return error; 2995 2996 if ((result <= 0) || (result > sc->num_tx_desc)) 2997 return EINVAL; 2998 2999 sc->tx_process_limit = result; 3000 3001 return 0; 3002 } /* ixv_sysctl_tx_process_limit */ 3003 3004 /************************************************************************ 3005 * ixv_sysctl_rx_process_limit 3006 ************************************************************************/ 3007 static int 3008 ixv_sysctl_rx_process_limit(SYSCTLFN_ARGS) 3009 { 3010 struct sysctlnode node = *rnode; 3011 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 3012 int error; 3013 int result = sc->rx_process_limit; 3014 3015 node.sysctl_data = &result; 3016 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3017 3018 if (error || newp == NULL) 3019 return error; 3020 3021 if ((result <= 0) || (result > sc->num_rx_desc)) 3022 return EINVAL; 3023 3024 sc->rx_process_limit = result; 3025 3026 return 0; 3027 } /* ixv_sysctl_rx_process_limit */ 3028 3029 /************************************************************************ 3030 * ixv_init_device_features 3031 ************************************************************************/ 3032 static void 3033 ixv_init_device_features(struct ixgbe_softc *sc) 3034 { 3035 sc->feat_cap = IXGBE_FEATURE_NETMAP 3036 | IXGBE_FEATURE_VF 3037 | IXGBE_FEATURE_RSS 3038 | IXGBE_FEATURE_LEGACY_TX; 3039 3040 /* A tad short on feature flags for VFs, atm. */ 3041 switch (sc->hw.mac.type) { 3042 case ixgbe_mac_82599_vf: 3043 break; 3044 case ixgbe_mac_X540_vf: 3045 break; 3046 case ixgbe_mac_X550_vf: 3047 case ixgbe_mac_X550EM_x_vf: 3048 case ixgbe_mac_X550EM_a_vf: 3049 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3050 break; 3051 default: 3052 break; 3053 } 3054 3055 /* Enabled by default... */ 3056 /* Is a virtual function (VF) */ 3057 if (sc->feat_cap & IXGBE_FEATURE_VF) 3058 sc->feat_en |= IXGBE_FEATURE_VF; 3059 /* Netmap */ 3060 if (sc->feat_cap & IXGBE_FEATURE_NETMAP) 3061 sc->feat_en |= IXGBE_FEATURE_NETMAP; 3062 /* Receive-Side Scaling (RSS) */ 3063 if (sc->feat_cap & IXGBE_FEATURE_RSS) 3064 sc->feat_en |= IXGBE_FEATURE_RSS; 3065 /* Needs advanced context descriptor regardless of offloads req'd */ 3066 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3067 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3068 3069 /* Enabled via sysctl... */ 3070 /* Legacy (single queue) transmit */ 3071 if ((sc->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3072 ixv_enable_legacy_tx) 3073 sc->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3074 } /* ixv_init_device_features */ 3075 3076 /************************************************************************ 3077 * ixv_shutdown - Shutdown entry point 3078 ************************************************************************/ 3079 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3080 static int 3081 ixv_shutdown(device_t dev) 3082 { 3083 struct ixgbe_softc *sc = device_private(dev); 3084 IXGBE_CORE_LOCK(sc); 3085 ixv_stop_locked(sc); 3086 IXGBE_CORE_UNLOCK(sc); 3087 3088 return (0); 3089 } /* ixv_shutdown */ 3090 #endif 3091 3092 static int 3093 ixv_ifflags_cb(struct ethercom *ec) 3094 { 3095 struct ifnet *ifp = &ec->ec_if; 3096 struct ixgbe_softc *sc = ifp->if_softc; 3097 u_short saved_flags; 3098 u_short change; 3099 int rv = 0; 3100 3101 IXGBE_CORE_LOCK(sc); 3102 3103 saved_flags = sc->if_flags; 3104 change = ifp->if_flags ^ sc->if_flags; 3105 if (change != 0) 3106 sc->if_flags = ifp->if_flags; 3107 3108 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3109 rv = ENETRESET; 3110 goto out; 3111 } else if ((change & IFF_PROMISC) != 0) { 3112 rv = ixv_set_rxfilter(sc); 3113 if (rv != 0) { 3114 /* Restore previous */ 3115 sc->if_flags = saved_flags; 3116 goto out; 3117 } 3118 } 3119 3120 /* Check for ec_capenable. */ 3121 change = ec->ec_capenable ^ sc->ec_capenable; 3122 sc->ec_capenable = ec->ec_capenable; 3123 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3124 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3125 rv = ENETRESET; 3126 goto out; 3127 } 3128 3129 /* 3130 * Special handling is not required for ETHERCAP_VLAN_MTU. 3131 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3132 */ 3133 3134 /* Set up VLAN support and filter */ 3135 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3136 rv = ixv_setup_vlan_support(sc); 3137 3138 out: 3139 IXGBE_CORE_UNLOCK(sc); 3140 3141 return rv; 3142 } 3143 3144 3145 /************************************************************************ 3146 * ixv_ioctl - Ioctl entry point 3147 * 3148 * Called when the user wants to configure the interface. 3149 * 3150 * return 0 on success, positive on failure 3151 ************************************************************************/ 3152 static int 3153 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3154 { 3155 struct ixgbe_softc *sc = ifp->if_softc; 3156 struct ixgbe_hw *hw = &sc->hw; 3157 struct ifcapreq *ifcr = data; 3158 int error; 3159 int l4csum_en; 3160 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3161 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3162 3163 switch (command) { 3164 case SIOCSIFFLAGS: 3165 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3166 break; 3167 case SIOCADDMULTI: { 3168 struct ether_multi *enm; 3169 struct ether_multistep step; 3170 struct ethercom *ec = &sc->osdep.ec; 3171 bool overflow = false; 3172 int mcnt = 0; 3173 3174 /* 3175 * Check the number of multicast address. If it exceeds, 3176 * return ENOSPC. 3177 * Update this code when we support API 1.3. 3178 */ 3179 ETHER_LOCK(ec); 3180 ETHER_FIRST_MULTI(step, ec, enm); 3181 while (enm != NULL) { 3182 mcnt++; 3183 3184 /* 3185 * This code is before adding, so one room is required 3186 * at least. 3187 */ 3188 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3189 overflow = true; 3190 break; 3191 } 3192 ETHER_NEXT_MULTI(step, enm); 3193 } 3194 ETHER_UNLOCK(ec); 3195 error = 0; 3196 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3197 error = hw->mac.ops.update_xcast_mode(hw, 3198 IXGBEVF_XCAST_MODE_ALLMULTI); 3199 if (error == IXGBE_ERR_NOT_TRUSTED) { 3200 device_printf(sc->dev, 3201 "this interface is not trusted\n"); 3202 error = EPERM; 3203 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3204 device_printf(sc->dev, 3205 "the PF doesn't support allmulti mode\n"); 3206 error = EOPNOTSUPP; 3207 } else if (error) { 3208 device_printf(sc->dev, 3209 "number of Ethernet multicast addresses " 3210 "exceeds the limit (%d). error = %d\n", 3211 IXGBE_MAX_VF_MC, error); 3212 error = ENOSPC; 3213 } else 3214 ec->ec_flags |= ETHER_F_ALLMULTI; 3215 } 3216 if (error) 3217 return error; 3218 } 3219 /*FALLTHROUGH*/ 3220 case SIOCDELMULTI: 3221 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3222 break; 3223 case SIOCSIFMEDIA: 3224 case SIOCGIFMEDIA: 3225 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3226 break; 3227 case SIOCSIFCAP: 3228 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3229 break; 3230 case SIOCSIFMTU: 3231 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3232 break; 3233 case SIOCZIFDATA: 3234 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3235 ixv_update_stats(sc); 3236 ixv_clear_evcnt(sc); 3237 break; 3238 default: 3239 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3240 break; 3241 } 3242 3243 switch (command) { 3244 case SIOCSIFCAP: 3245 /* Layer-4 Rx checksum offload has to be turned on and 3246 * off as a unit. 3247 */ 3248 l4csum_en = ifcr->ifcr_capenable & l4csum; 3249 if (l4csum_en != l4csum && l4csum_en != 0) 3250 return EINVAL; 3251 /*FALLTHROUGH*/ 3252 case SIOCADDMULTI: 3253 case SIOCDELMULTI: 3254 case SIOCSIFFLAGS: 3255 case SIOCSIFMTU: 3256 default: 3257 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3258 return error; 3259 if ((ifp->if_flags & IFF_RUNNING) == 0) 3260 ; 3261 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3262 IXGBE_CORE_LOCK(sc); 3263 ixv_init_locked(sc); 3264 IXGBE_CORE_UNLOCK(sc); 3265 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3266 /* 3267 * Multicast list has changed; set the hardware filter 3268 * accordingly. 3269 */ 3270 IXGBE_CORE_LOCK(sc); 3271 ixv_disable_intr(sc); 3272 ixv_set_rxfilter(sc); 3273 ixv_enable_intr(sc); 3274 IXGBE_CORE_UNLOCK(sc); 3275 } 3276 return 0; 3277 } 3278 } /* ixv_ioctl */ 3279 3280 /************************************************************************ 3281 * ixv_init 3282 ************************************************************************/ 3283 static int 3284 ixv_init(struct ifnet *ifp) 3285 { 3286 struct ixgbe_softc *sc = ifp->if_softc; 3287 3288 IXGBE_CORE_LOCK(sc); 3289 ixv_init_locked(sc); 3290 IXGBE_CORE_UNLOCK(sc); 3291 3292 return 0; 3293 } /* ixv_init */ 3294 3295 /************************************************************************ 3296 * ixv_handle_que 3297 ************************************************************************/ 3298 static void 3299 ixv_handle_que(void *context) 3300 { 3301 struct ix_queue *que = context; 3302 struct ixgbe_softc *sc = que->sc; 3303 struct tx_ring *txr = que->txr; 3304 struct ifnet *ifp = sc->ifp; 3305 bool more; 3306 3307 IXGBE_EVC_ADD(&que->handleq, 1); 3308 3309 if (ifp->if_flags & IFF_RUNNING) { 3310 IXGBE_TX_LOCK(txr); 3311 more = ixgbe_txeof(txr); 3312 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3313 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3314 ixgbe_mq_start_locked(ifp, txr); 3315 /* Only for queue 0 */ 3316 /* NetBSD still needs this for CBQ */ 3317 if ((&sc->queues[0] == que) 3318 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3319 ixgbe_legacy_start_locked(ifp, txr); 3320 IXGBE_TX_UNLOCK(txr); 3321 more |= ixgbe_rxeof(que); 3322 if (more) { 3323 IXGBE_EVC_ADD(&que->req, 1); 3324 if (sc->txrx_use_workqueue) { 3325 /* 3326 * "enqueued flag" is not required here 3327 * the same as ixg(4). See ixgbe_msix_que(). 3328 */ 3329 workqueue_enqueue(sc->que_wq, 3330 &que->wq_cookie, curcpu()); 3331 } else 3332 softint_schedule(que->que_si); 3333 return; 3334 } 3335 } 3336 3337 /* Re-enable this interrupt */ 3338 ixv_enable_queue(sc, que->msix); 3339 3340 return; 3341 } /* ixv_handle_que */ 3342 3343 /************************************************************************ 3344 * ixv_handle_que_work 3345 ************************************************************************/ 3346 static void 3347 ixv_handle_que_work(struct work *wk, void *context) 3348 { 3349 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3350 3351 /* 3352 * "enqueued flag" is not required here the same as ixg(4). 3353 * See ixgbe_msix_que(). 3354 */ 3355 ixv_handle_que(que); 3356 } 3357 3358 /************************************************************************ 3359 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3360 ************************************************************************/ 3361 static int 3362 ixv_allocate_msix(struct ixgbe_softc *sc, const struct pci_attach_args *pa) 3363 { 3364 device_t dev = sc->dev; 3365 struct ix_queue *que = sc->queues; 3366 struct tx_ring *txr = sc->tx_rings; 3367 int error, msix_ctrl, rid, vector = 0; 3368 pci_chipset_tag_t pc; 3369 pcitag_t tag; 3370 char intrbuf[PCI_INTRSTR_LEN]; 3371 char wqname[MAXCOMLEN]; 3372 char intr_xname[32]; 3373 const char *intrstr = NULL; 3374 kcpuset_t *affinity; 3375 int cpu_id = 0; 3376 3377 pc = sc->osdep.pc; 3378 tag = sc->osdep.tag; 3379 3380 sc->osdep.nintrs = sc->num_queues + 1; 3381 if (pci_msix_alloc_exact(pa, &sc->osdep.intrs, 3382 sc->osdep.nintrs) != 0) { 3383 aprint_error_dev(dev, 3384 "failed to allocate MSI-X interrupt\n"); 3385 return (ENXIO); 3386 } 3387 3388 kcpuset_create(&affinity, false); 3389 for (int i = 0; i < sc->num_queues; i++, vector++, que++, txr++) { 3390 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3391 device_xname(dev), i); 3392 intrstr = pci_intr_string(pc, sc->osdep.intrs[i], intrbuf, 3393 sizeof(intrbuf)); 3394 pci_intr_setattr(pc, &sc->osdep.intrs[i], PCI_INTR_MPSAFE, 3395 true); 3396 3397 /* Set the handler function */ 3398 que->res = sc->osdep.ihs[i] = pci_intr_establish_xname(pc, 3399 sc->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3400 intr_xname); 3401 if (que->res == NULL) { 3402 pci_intr_release(pc, sc->osdep.intrs, 3403 sc->osdep.nintrs); 3404 aprint_error_dev(dev, 3405 "Failed to register QUE handler\n"); 3406 kcpuset_destroy(affinity); 3407 return (ENXIO); 3408 } 3409 que->msix = vector; 3410 sc->active_queues |= (u64)(1 << que->msix); 3411 3412 cpu_id = i; 3413 /* Round-robin affinity */ 3414 kcpuset_zero(affinity); 3415 kcpuset_set(affinity, cpu_id % ncpu); 3416 error = interrupt_distribute(sc->osdep.ihs[i], affinity, NULL); 3417 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3418 intrstr); 3419 if (error == 0) 3420 aprint_normal(", bound queue %d to cpu %d\n", 3421 i, cpu_id % ncpu); 3422 else 3423 aprint_normal("\n"); 3424 3425 #ifndef IXGBE_LEGACY_TX 3426 txr->txr_si 3427 = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 3428 ixgbe_deferred_mq_start, txr); 3429 #endif 3430 que->que_si 3431 = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 3432 ixv_handle_que, que); 3433 if (que->que_si == NULL) { 3434 aprint_error_dev(dev, 3435 "could not establish software interrupt\n"); 3436 } 3437 } 3438 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3439 error = workqueue_create(&sc->txr_wq, wqname, 3440 ixgbe_deferred_mq_start_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3441 WQ_PERCPU | WQ_MPSAFE); 3442 if (error) { 3443 aprint_error_dev(dev, 3444 "couldn't create workqueue for deferred Tx\n"); 3445 } 3446 sc->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3447 3448 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3449 error = workqueue_create(&sc->que_wq, wqname, 3450 ixv_handle_que_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3451 WQ_PERCPU | WQ_MPSAFE); 3452 if (error) { 3453 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3454 } 3455 3456 /* and Mailbox */ 3457 cpu_id++; 3458 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3459 sc->vector = vector; 3460 intrstr = pci_intr_string(pc, sc->osdep.intrs[vector], intrbuf, 3461 sizeof(intrbuf)); 3462 pci_intr_setattr(pc, &sc->osdep.intrs[vector], PCI_INTR_MPSAFE, true); 3463 3464 /* Set the mbx handler function */ 3465 sc->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3466 sc->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, sc, intr_xname); 3467 if (sc->osdep.ihs[vector] == NULL) { 3468 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3469 kcpuset_destroy(affinity); 3470 return (ENXIO); 3471 } 3472 /* Round-robin affinity */ 3473 kcpuset_zero(affinity); 3474 kcpuset_set(affinity, cpu_id % ncpu); 3475 error = interrupt_distribute(sc->osdep.ihs[vector], affinity, 3476 NULL); 3477 3478 aprint_normal_dev(dev, 3479 "for link, interrupting at %s", intrstr); 3480 if (error == 0) 3481 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3482 else 3483 aprint_normal("\n"); 3484 3485 /* Tasklets for Mailbox */ 3486 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3487 error = workqueue_create(&sc->admin_wq, wqname, 3488 ixv_handle_admin, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE); 3489 if (error) { 3490 aprint_error_dev(dev, 3491 "could not create admin workqueue (%d)\n", error); 3492 goto err_out; 3493 } 3494 3495 /* 3496 * Due to a broken design QEMU will fail to properly 3497 * enable the guest for MSI-X unless the vectors in 3498 * the table are all set up, so we must rewrite the 3499 * ENABLE in the MSI-X control register again at this 3500 * point to cause it to successfully initialize us. 3501 */ 3502 if (sc->hw.mac.type == ixgbe_mac_82599_vf) { 3503 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3504 rid += PCI_MSIX_CTL; 3505 msix_ctrl = pci_conf_read(pc, tag, rid); 3506 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3507 pci_conf_write(pc, tag, rid, msix_ctrl); 3508 } 3509 3510 kcpuset_destroy(affinity); 3511 return (0); 3512 err_out: 3513 kcpuset_destroy(affinity); 3514 ixv_free_deferred_handlers(sc); 3515 ixv_free_pci_resources(sc); 3516 return (error); 3517 } /* ixv_allocate_msix */ 3518 3519 /************************************************************************ 3520 * ixv_configure_interrupts - Setup MSI-X resources 3521 * 3522 * Note: The VF device MUST use MSI-X, there is no fallback. 3523 ************************************************************************/ 3524 static int 3525 ixv_configure_interrupts(struct ixgbe_softc *sc) 3526 { 3527 device_t dev = sc->dev; 3528 int want, queues, msgs; 3529 3530 /* Must have at least 2 MSI-X vectors */ 3531 msgs = pci_msix_count(sc->osdep.pc, sc->osdep.tag); 3532 if (msgs < 2) { 3533 aprint_error_dev(dev, "MSIX config error\n"); 3534 return (ENXIO); 3535 } 3536 msgs = MIN(msgs, IXG_MAX_NINTR); 3537 3538 /* Figure out a reasonable auto config value */ 3539 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3540 3541 if (ixv_num_queues != 0) 3542 queues = ixv_num_queues; 3543 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3544 queues = IXGBE_VF_MAX_TX_QUEUES; 3545 3546 /* 3547 * Want vectors for the queues, 3548 * plus an additional for mailbox. 3549 */ 3550 want = queues + 1; 3551 if (msgs >= want) 3552 msgs = want; 3553 else { 3554 aprint_error_dev(dev, 3555 "MSI-X Configuration Problem, " 3556 "%d vectors but %d queues wanted!\n", msgs, want); 3557 return -1; 3558 } 3559 3560 aprint_normal_dev(dev, 3561 "Using MSI-X interrupts with %d vectors\n", msgs); 3562 sc->num_queues = queues; 3563 3564 return (0); 3565 } /* ixv_configure_interrupts */ 3566 3567 3568 /************************************************************************ 3569 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3570 * 3571 * Done outside of interrupt context since the driver might sleep 3572 ************************************************************************/ 3573 static void 3574 ixv_handle_admin(struct work *wk, void *context) 3575 { 3576 struct ixgbe_softc *sc = context; 3577 struct ixgbe_hw *hw = &sc->hw; 3578 3579 IXGBE_CORE_LOCK(sc); 3580 3581 IXGBE_EVC_ADD(&sc->link_workev, 1); 3582 sc->hw.mac.ops.check_link(&sc->hw, &sc->link_speed, 3583 &sc->link_up, FALSE); 3584 ixv_update_link_status(sc); 3585 3586 sc->task_requests = 0; 3587 atomic_store_relaxed(&sc->admin_pending, 0); 3588 3589 /* Re-enable interrupts */ 3590 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 3591 3592 IXGBE_CORE_UNLOCK(sc); 3593 } /* ixv_handle_admin */ 3594 3595 /************************************************************************ 3596 * ixv_check_link - Used in the local timer to poll for link changes 3597 ************************************************************************/ 3598 static s32 3599 ixv_check_link(struct ixgbe_softc *sc) 3600 { 3601 s32 error; 3602 3603 KASSERT(mutex_owned(&sc->core_mtx)); 3604 3605 sc->hw.mac.get_link_status = TRUE; 3606 3607 error = sc->hw.mac.ops.check_link(&sc->hw, 3608 &sc->link_speed, &sc->link_up, FALSE); 3609 ixv_update_link_status(sc); 3610 3611 return error; 3612 } /* ixv_check_link */ 3613