1 /****************************************************************************** 2 3 Copyright (c) 2001-2011, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /* 34 * Copyright (c) 2011 The NetBSD Foundation, Inc. 35 * All rights reserved. 36 * 37 * This code is derived from software contributed to The NetBSD Foundation 38 * by Coyote Point Systems, Inc. 39 * 40 * Redistribution and use in source and binary forms, with or without 41 * modification, are permitted provided that the following conditions 42 * are met: 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in the 47 * documentation and/or other materials provided with the distribution. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 50 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 51 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 52 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 53 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 */ 61 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.51 2011/04/25 23:34:21 jfv Exp $*/ 62 /*$NetBSD: ixgbe.c,v 1.5 2012/06/18 06:21:11 dsl Exp $*/ 63 64 #include "opt_inet.h" 65 66 #include "ixgbe.h" 67 68 /********************************************************************* 69 * Set this to one to display debug statistics 70 *********************************************************************/ 71 int ixgbe_display_debug_stats = 0; 72 73 /********************************************************************* 74 * Driver version 75 *********************************************************************/ 76 char ixgbe_driver_version[] = "2.3.10"; 77 78 /********************************************************************* 79 * PCI Device ID Table 80 * 81 * Used by probe to select devices to load on 82 * Last field stores an index into ixgbe_strings 83 * Last entry must be all 0s 84 * 85 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 86 *********************************************************************/ 87 88 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 89 { 90 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 91 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 92 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 93 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 94 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 95 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 96 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 97 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 98 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 99 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 110 /* required last entry */ 111 {0, 0, 0, 0, 0} 112 }; 113 114 /********************************************************************* 115 * Table of branding strings 116 *********************************************************************/ 117 118 static const char *ixgbe_strings[] = { 119 "Intel(R) PRO/10GbE PCI-Express Network Driver" 120 }; 121 122 /********************************************************************* 123 * Function prototypes 124 *********************************************************************/ 125 static int ixgbe_probe(device_t, cfdata_t, void *); 126 static void ixgbe_attach(device_t, device_t, void *); 127 static int ixgbe_detach(device_t, int); 128 #if 0 129 static int ixgbe_shutdown(device_t); 130 #endif 131 static void ixgbe_start(struct ifnet *); 132 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 133 #if __FreeBSD_version >= 800000 134 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); 135 static int ixgbe_mq_start_locked(struct ifnet *, 136 struct tx_ring *, struct mbuf *); 137 static void ixgbe_qflush(struct ifnet *); 138 #endif 139 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 140 static void ixgbe_ifstop(struct ifnet *, int); 141 static int ixgbe_init(struct ifnet *); 142 static void ixgbe_init_locked(struct adapter *); 143 static void ixgbe_stop(void *); 144 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 145 static int ixgbe_media_change(struct ifnet *); 146 static void ixgbe_identify_hardware(struct adapter *); 147 static int ixgbe_allocate_pci_resources(struct adapter *, 148 const struct pci_attach_args *); 149 static int ixgbe_allocate_msix(struct adapter *, 150 const struct pci_attach_args *); 151 static int ixgbe_allocate_legacy(struct adapter *, 152 const struct pci_attach_args *); 153 static int ixgbe_allocate_queues(struct adapter *); 154 static int ixgbe_setup_msix(struct adapter *); 155 static void ixgbe_free_pci_resources(struct adapter *); 156 static void ixgbe_local_timer(void *); 157 static int ixgbe_setup_interface(device_t, struct adapter *); 158 static void ixgbe_config_link(struct adapter *); 159 160 static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 161 static int ixgbe_setup_transmit_structures(struct adapter *); 162 static void ixgbe_setup_transmit_ring(struct tx_ring *); 163 static void ixgbe_initialize_transmit_units(struct adapter *); 164 static void ixgbe_free_transmit_structures(struct adapter *); 165 static void ixgbe_free_transmit_buffers(struct tx_ring *); 166 167 static int ixgbe_allocate_receive_buffers(struct rx_ring *); 168 static int ixgbe_setup_receive_structures(struct adapter *); 169 static int ixgbe_setup_receive_ring(struct rx_ring *); 170 static void ixgbe_initialize_receive_units(struct adapter *); 171 static void ixgbe_free_receive_structures(struct adapter *); 172 static void ixgbe_free_receive_buffers(struct rx_ring *); 173 static void ixgbe_setup_hw_rsc(struct rx_ring *); 174 175 static void ixgbe_enable_intr(struct adapter *); 176 static void ixgbe_disable_intr(struct adapter *); 177 static void ixgbe_update_stats_counters(struct adapter *); 178 static bool ixgbe_txeof(struct tx_ring *); 179 static bool ixgbe_rxeof(struct ix_queue *, int); 180 static void ixgbe_rx_checksum(u32, struct mbuf *, u32, 181 struct ixgbe_hw_stats *); 182 static void ixgbe_set_promisc(struct adapter *); 183 static void ixgbe_set_multi(struct adapter *); 184 static void ixgbe_update_link_status(struct adapter *); 185 static void ixgbe_refresh_mbufs(struct rx_ring *, int); 186 static int ixgbe_xmit(struct tx_ring *, struct mbuf *); 187 static int ixgbe_set_flowcntl(SYSCTLFN_PROTO); 188 static int ixgbe_set_advertise(SYSCTLFN_PROTO); 189 static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 190 struct ixgbe_dma_alloc *, int); 191 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 192 static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 193 const char *, int *, int); 194 static u32 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 195 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 196 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 197 static void ixgbe_configure_ivars(struct adapter *); 198 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 199 200 static void ixgbe_setup_vlan_hw_support(struct adapter *); 201 #if 0 202 static void ixgbe_register_vlan(void *, struct ifnet *, u16); 203 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 204 #endif 205 206 static void ixgbe_add_hw_stats(struct adapter *adapter); 207 208 static __inline void ixgbe_rx_discard(struct rx_ring *, int); 209 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 210 struct mbuf *, u32); 211 212 /* Support for pluggable optic modules */ 213 static bool ixgbe_sfp_probe(struct adapter *); 214 static void ixgbe_setup_optics(struct adapter *); 215 216 /* Legacy (single vector interrupt handler */ 217 static int ixgbe_legacy_irq(void *); 218 219 #if defined(NETBSD_MSI_OR_MSIX) 220 /* The MSI/X Interrupt handlers */ 221 static void ixgbe_msix_que(void *); 222 static void ixgbe_msix_link(void *); 223 #endif 224 225 /* Software interrupts for deferred work */ 226 static void ixgbe_handle_que(void *); 227 static void ixgbe_handle_link(void *); 228 static void ixgbe_handle_msf(void *); 229 static void ixgbe_handle_mod(void *); 230 231 const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); 232 static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 233 234 #ifdef IXGBE_FDIR 235 static void ixgbe_atr(struct tx_ring *, struct mbuf *); 236 static void ixgbe_reinit_fdir(void *, int); 237 #endif 238 239 /********************************************************************* 240 * FreeBSD Device Interface Entry Points 241 *********************************************************************/ 242 243 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 244 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 245 DVF_DETACH_SHUTDOWN); 246 247 #if 0 248 devclass_t ixgbe_devclass; 249 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 250 251 MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 252 MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 253 #endif 254 255 /* 256 ** TUNEABLE PARAMETERS: 257 */ 258 259 /* 260 ** AIM: Adaptive Interrupt Moderation 261 ** which means that the interrupt rate 262 ** is varied over time based on the 263 ** traffic for that interrupt vector 264 */ 265 static int ixgbe_enable_aim = TRUE; 266 #define TUNABLE_INT(__x, __y) 267 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 268 269 static int ixgbe_max_interrupt_rate = (8000000 / IXGBE_LOW_LATENCY); 270 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate); 271 272 /* How many packets rxeof tries to clean at a time */ 273 static int ixgbe_rx_process_limit = 256; 274 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 275 276 /* Flow control setting, default to full */ 277 static int ixgbe_flow_control = ixgbe_fc_full; 278 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 279 280 /* 281 ** Smart speed setting, default to on 282 ** this only works as a compile option 283 ** right now as its during attach, set 284 ** this to 'ixgbe_smart_speed_off' to 285 ** disable. 286 */ 287 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 288 289 /* 290 * MSIX should be the default for best performance, 291 * but this allows it to be forced off for testing. 292 */ 293 static int ixgbe_enable_msix = 1; 294 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 295 296 /* 297 * Header split: this causes the hardware to DMA 298 * the header into a separate mbuf from the payload, 299 * it can be a performance win in some workloads, but 300 * in others it actually hurts, its off by default. 301 */ 302 static bool ixgbe_header_split = FALSE; 303 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); 304 305 #if defined(NETBSD_MSI_OR_MSIX) 306 /* 307 * Number of Queues, can be set to 0, 308 * it then autoconfigures based on the 309 * number of cpus with a max of 8. This 310 * can be overriden manually here. 311 */ 312 static int ixgbe_num_queues = 0; 313 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); 314 #endif 315 316 /* 317 ** Number of TX descriptors per ring, 318 ** setting higher than RX as this seems 319 ** the better performing choice. 320 */ 321 static int ixgbe_txd = PERFORM_TXD; 322 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 323 324 /* Number of RX descriptors per ring */ 325 static int ixgbe_rxd = PERFORM_RXD; 326 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 327 328 /* Keep running tab on them for sanity check */ 329 static int ixgbe_total_ports; 330 331 #ifdef IXGBE_FDIR 332 /* 333 ** For Flow Director: this is the 334 ** number of TX packets we sample 335 ** for the filter pool, this means 336 ** every 20th packet will be probed. 337 ** 338 ** This feature can be disabled by 339 ** setting this to 0. 340 */ 341 static int atr_sample_rate = 20; 342 /* 343 ** Flow Director actually 'steals' 344 ** part of the packet buffer as its 345 ** filter pool, this variable controls 346 ** how much it uses: 347 ** 0 = 64K, 1 = 128K, 2 = 256K 348 */ 349 static int fdir_pballoc = 1; 350 #endif 351 352 /********************************************************************* 353 * Device identification routine 354 * 355 * ixgbe_probe determines if the driver should be loaded on 356 * adapter based on PCI vendor/device id of the adapter. 357 * 358 * return 1 on success, 0 on failure 359 *********************************************************************/ 360 361 static int 362 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 363 { 364 const struct pci_attach_args *pa = aux; 365 366 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 367 } 368 369 static ixgbe_vendor_info_t * 370 ixgbe_lookup(const struct pci_attach_args *pa) 371 { 372 pcireg_t subid; 373 ixgbe_vendor_info_t *ent; 374 375 INIT_DEBUGOUT("ixgbe_probe: begin"); 376 377 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 378 return NULL; 379 380 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 381 382 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 383 if (PCI_VENDOR(pa->pa_id) == ent->vendor_id && 384 PCI_PRODUCT(pa->pa_id) == ent->device_id && 385 386 (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id || 387 ent->subvendor_id == 0) && 388 389 (PCI_SUBSYS_ID(subid) == ent->subdevice_id || 390 ent->subdevice_id == 0)) { 391 ++ixgbe_total_ports; 392 return ent; 393 } 394 } 395 return NULL; 396 } 397 398 399 static void 400 ixgbe_sysctl_attach(struct adapter *adapter) 401 { 402 struct sysctllog **log; 403 const struct sysctlnode *rnode, *cnode; 404 device_t dev; 405 406 dev = adapter->dev; 407 log = &adapter->sysctllog; 408 409 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 410 aprint_error_dev(dev, "could not create sysctl root\n"); 411 return; 412 } 413 414 if (sysctl_createv(log, 0, &rnode, &cnode, 415 CTLFLAG_READONLY, CTLTYPE_INT, 416 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 417 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 418 aprint_error_dev(dev, "could not create sysctl\n"); 419 420 if (sysctl_createv(log, 0, &rnode, &cnode, 421 CTLFLAG_READONLY, CTLTYPE_INT, 422 "num_queues", SYSCTL_DESCR("Number of queues"), 423 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 424 aprint_error_dev(dev, "could not create sysctl\n"); 425 426 if (sysctl_createv(log, 0, &rnode, &cnode, 427 CTLFLAG_READWRITE, CTLTYPE_INT, 428 "flow_control", SYSCTL_DESCR("Flow Control"), 429 ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 430 aprint_error_dev(dev, "could not create sysctl\n"); 431 432 if (sysctl_createv(log, 0, &rnode, &cnode, 433 CTLFLAG_READWRITE, CTLTYPE_INT, 434 "advertise_gig", SYSCTL_DESCR("1G Link"), 435 ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 436 aprint_error_dev(dev, "could not create sysctl\n"); 437 438 /* XXX This is an *instance* sysctl controlling a *global* variable. 439 * XXX It's that way in the FreeBSD driver that this derives from. 440 */ 441 if (sysctl_createv(log, 0, &rnode, &cnode, 442 CTLFLAG_READWRITE, CTLTYPE_INT, 443 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 444 NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 445 aprint_error_dev(dev, "could not create sysctl\n"); 446 } 447 448 /********************************************************************* 449 * Device initialization routine 450 * 451 * The attach entry point is called when the driver is being loaded. 452 * This routine identifies the type of hardware, allocates all resources 453 * and initializes the hardware. 454 * 455 * return 0 on success, positive on failure 456 *********************************************************************/ 457 458 static void 459 ixgbe_attach(device_t parent, device_t dev, void *aux) 460 { 461 struct adapter *adapter; 462 struct ixgbe_hw *hw; 463 int error = 0; 464 u16 csum; 465 u32 ctrl_ext; 466 ixgbe_vendor_info_t *ent; 467 const struct pci_attach_args *pa = aux; 468 469 INIT_DEBUGOUT("ixgbe_attach: begin"); 470 471 /* Allocate, clear, and link in our adapter structure */ 472 adapter = device_private(dev); 473 adapter->dev = adapter->osdep.dev = dev; 474 hw = &adapter->hw; 475 adapter->osdep.pc = pa->pa_pc; 476 adapter->osdep.tag = pa->pa_tag; 477 adapter->osdep.dmat = pa->pa_dmat; 478 479 ent = ixgbe_lookup(pa); 480 481 KASSERT(ent != NULL); 482 483 aprint_normal(": %s, Version - %s\n", 484 ixgbe_strings[ent->index], ixgbe_driver_version); 485 486 /* Core Lock Init*/ 487 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 488 489 /* SYSCTL APIs */ 490 491 ixgbe_sysctl_attach(adapter); 492 493 /* Set up the timer callout */ 494 callout_init(&adapter->timer, 0); 495 496 /* Determine hardware revision */ 497 ixgbe_identify_hardware(adapter); 498 499 /* Do base PCI setup - map BAR0 */ 500 if (ixgbe_allocate_pci_resources(adapter, pa)) { 501 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 502 error = ENXIO; 503 goto err_out; 504 } 505 506 /* Do descriptor calc and sanity checks */ 507 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 508 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 509 aprint_error_dev(dev, "TXD config issue, using default!\n"); 510 adapter->num_tx_desc = DEFAULT_TXD; 511 } else 512 adapter->num_tx_desc = ixgbe_txd; 513 514 /* 515 ** With many RX rings it is easy to exceed the 516 ** system mbuf allocation. Tuning nmbclusters 517 ** can alleviate this. 518 */ 519 if (nmbclusters > 0 ) { 520 int s; 521 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; 522 if (s > nmbclusters) { 523 aprint_error_dev(dev, "RX Descriptors exceed " 524 "system mbuf max, using default instead!\n"); 525 ixgbe_rxd = DEFAULT_RXD; 526 } 527 } 528 529 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 530 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 531 aprint_error_dev(dev, "RXD config issue, using default!\n"); 532 adapter->num_rx_desc = DEFAULT_RXD; 533 } else 534 adapter->num_rx_desc = ixgbe_rxd; 535 536 /* Allocate our TX/RX Queues */ 537 if (ixgbe_allocate_queues(adapter)) { 538 error = ENOMEM; 539 goto err_out; 540 } 541 542 /* Allocate multicast array memory. */ 543 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 544 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 545 if (adapter->mta == NULL) { 546 aprint_error_dev(dev, "Cannot allocate multicast setup array\n"); 547 error = ENOMEM; 548 goto err_late; 549 } 550 551 /* Initialize the shared code */ 552 error = ixgbe_init_shared_code(hw); 553 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 554 /* 555 ** No optics in this port, set up 556 ** so the timer routine will probe 557 ** for later insertion. 558 */ 559 adapter->sfp_probe = TRUE; 560 error = 0; 561 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 562 aprint_error_dev(dev,"Unsupported SFP+ module detected!\n"); 563 error = EIO; 564 goto err_late; 565 } else if (error) { 566 aprint_error_dev(dev,"Unable to initialize the shared code\n"); 567 error = EIO; 568 goto err_late; 569 } 570 571 /* Make sure we have a good EEPROM before we read from it */ 572 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 573 aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n"); 574 error = EIO; 575 goto err_late; 576 } 577 578 /* Get Hardware Flow Control setting */ 579 hw->fc.requested_mode = ixgbe_fc_full; 580 hw->fc.pause_time = IXGBE_FC_PAUSE; 581 hw->fc.low_water = IXGBE_FC_LO; 582 hw->fc.high_water = IXGBE_FC_HI; 583 hw->fc.send_xon = TRUE; 584 585 error = ixgbe_init_hw(hw); 586 if (error == IXGBE_ERR_EEPROM_VERSION) { 587 aprint_error_dev(dev, "This device is a pre-production adapter/" 588 "LOM. Please be aware there may be issues associated " 589 "with your hardware.\n If you are experiencing problems " 590 "please contact your Intel or hardware representative " 591 "who provided you with this hardware.\n"); 592 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 593 aprint_error_dev(dev,"Unsupported SFP+ Module\n"); 594 595 if (error) { 596 error = EIO; 597 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 598 goto err_late; 599 } 600 601 /* Detect and set physical type */ 602 ixgbe_setup_optics(adapter); 603 604 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 605 error = ixgbe_allocate_msix(adapter, pa); 606 else 607 error = ixgbe_allocate_legacy(adapter, pa); 608 if (error) 609 goto err_late; 610 611 /* Setup OS specific network interface */ 612 if (ixgbe_setup_interface(dev, adapter) != 0) 613 goto err_late; 614 615 /* Sysctl for limiting the amount of work done in software interrupts */ 616 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 617 "max number of rx packets to process", &adapter->rx_process_limit, 618 ixgbe_rx_process_limit); 619 620 /* Initialize statistics */ 621 ixgbe_update_stats_counters(adapter); 622 623 /* Print PCIE bus type/speed/width info */ 624 ixgbe_get_bus_info(hw); 625 aprint_normal_dev(dev,"PCI Express Bus: Speed %s %s\n", 626 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": 627 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), 628 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : 629 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : 630 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : 631 ("Unknown")); 632 633 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 634 (hw->bus.speed == ixgbe_bus_speed_2500)) { 635 aprint_error_dev(dev, "PCI-Express bandwidth available" 636 " for this card\n is not sufficient for" 637 " optimal performance.\n"); 638 aprint_error_dev(dev, "For optimal performance a x8 " 639 "PCIE, or x4 PCIE 2 slot is required.\n"); 640 } 641 642 /* let hardware know driver is loaded */ 643 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 644 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 645 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 646 647 ixgbe_add_hw_stats(adapter); 648 649 INIT_DEBUGOUT("ixgbe_attach: end"); 650 return; 651 err_late: 652 ixgbe_free_transmit_structures(adapter); 653 ixgbe_free_receive_structures(adapter); 654 err_out: 655 if (adapter->ifp != NULL) 656 if_free(adapter->ifp); 657 ixgbe_free_pci_resources(adapter); 658 if (adapter->mta != NULL) 659 free(adapter->mta, M_DEVBUF); 660 return; 661 662 } 663 664 /********************************************************************* 665 * Device removal routine 666 * 667 * The detach entry point is called when the driver is being removed. 668 * This routine stops the adapter and deallocates all the resources 669 * that were allocated for driver operation. 670 * 671 * return 0 on success, positive on failure 672 *********************************************************************/ 673 674 static int 675 ixgbe_detach(device_t dev, int flags) 676 { 677 struct adapter *adapter = device_private(dev); 678 struct tx_ring *txr = adapter->tx_rings; 679 struct rx_ring *rxr = adapter->rx_rings; 680 struct ixgbe_hw_stats *stats = &adapter->stats; 681 struct ix_queue *que = adapter->queues; 682 u32 ctrl_ext; 683 684 INIT_DEBUGOUT("ixgbe_detach: begin"); 685 686 /* Make sure VLANs are not using driver */ 687 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 688 ; /* nothing to do: no VLANs */ 689 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 690 vlan_ifdetach(adapter->ifp); 691 else { 692 aprint_error_dev(dev, "VLANs in use\n"); 693 return EBUSY; 694 } 695 696 IXGBE_CORE_LOCK(adapter); 697 ixgbe_stop(adapter); 698 IXGBE_CORE_UNLOCK(adapter); 699 700 for (int i = 0; i < adapter->num_queues; i++, que++) { 701 softint_disestablish(que->que_si); 702 } 703 704 /* Drain the Link queue */ 705 softint_disestablish(adapter->link_si); 706 softint_disestablish(adapter->mod_si); 707 softint_disestablish(adapter->msf_si); 708 #ifdef IXGBE_FDIR 709 softint_disestablish(adapter->fdir_si); 710 #endif 711 712 /* let hardware know driver is unloading */ 713 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 714 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 715 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 716 717 ether_ifdetach(adapter->ifp); 718 callout_halt(&adapter->timer, NULL); 719 ixgbe_free_pci_resources(adapter); 720 #if 0 /* XXX the NetBSD port is probably missing something here */ 721 bus_generic_detach(dev); 722 #endif 723 if_detach(adapter->ifp); 724 725 sysctl_teardown(&adapter->sysctllog); 726 evcnt_detach(&adapter->handleq); 727 evcnt_detach(&adapter->req); 728 evcnt_detach(&adapter->morerx); 729 evcnt_detach(&adapter->moretx); 730 evcnt_detach(&adapter->txloops); 731 evcnt_detach(&adapter->efbig_tx_dma_setup); 732 evcnt_detach(&adapter->m_defrag_failed); 733 evcnt_detach(&adapter->efbig2_tx_dma_setup); 734 evcnt_detach(&adapter->einval_tx_dma_setup); 735 evcnt_detach(&adapter->other_tx_dma_setup); 736 evcnt_detach(&adapter->eagain_tx_dma_setup); 737 evcnt_detach(&adapter->enomem_tx_dma_setup); 738 evcnt_detach(&adapter->watchdog_events); 739 evcnt_detach(&adapter->tso_err); 740 evcnt_detach(&adapter->tso_tx); 741 evcnt_detach(&adapter->link_irq); 742 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 743 evcnt_detach(&txr->no_desc_avail); 744 evcnt_detach(&txr->total_packets); 745 746 if (i < __arraycount(adapter->stats.mpc)) { 747 evcnt_detach(&adapter->stats.mpc[i]); 748 } 749 if (i < __arraycount(adapter->stats.pxontxc)) { 750 evcnt_detach(&adapter->stats.pxontxc[i]); 751 evcnt_detach(&adapter->stats.pxonrxc[i]); 752 evcnt_detach(&adapter->stats.pxofftxc[i]); 753 evcnt_detach(&adapter->stats.pxoffrxc[i]); 754 evcnt_detach(&adapter->stats.pxon2offc[i]); 755 } 756 if (i < __arraycount(adapter->stats.qprc)) { 757 evcnt_detach(&adapter->stats.qprc[i]); 758 evcnt_detach(&adapter->stats.qptc[i]); 759 evcnt_detach(&adapter->stats.qbrc[i]); 760 evcnt_detach(&adapter->stats.qbtc[i]); 761 evcnt_detach(&adapter->stats.qprdc[i]); 762 } 763 764 evcnt_detach(&rxr->rx_packets); 765 evcnt_detach(&rxr->rx_bytes); 766 evcnt_detach(&rxr->no_jmbuf); 767 evcnt_detach(&rxr->rx_discarded); 768 evcnt_detach(&rxr->rx_split_packets); 769 evcnt_detach(&rxr->rx_irq); 770 } 771 evcnt_detach(&stats->ipcs); 772 evcnt_detach(&stats->l4cs); 773 evcnt_detach(&stats->ipcs_bad); 774 evcnt_detach(&stats->l4cs_bad); 775 evcnt_detach(&stats->intzero); 776 evcnt_detach(&stats->legint); 777 evcnt_detach(&stats->crcerrs); 778 evcnt_detach(&stats->illerrc); 779 evcnt_detach(&stats->errbc); 780 evcnt_detach(&stats->mspdc); 781 evcnt_detach(&stats->mlfc); 782 evcnt_detach(&stats->mrfc); 783 evcnt_detach(&stats->rlec); 784 evcnt_detach(&stats->lxontxc); 785 evcnt_detach(&stats->lxonrxc); 786 evcnt_detach(&stats->lxofftxc); 787 evcnt_detach(&stats->lxoffrxc); 788 789 /* Packet Reception Stats */ 790 evcnt_detach(&stats->tor); 791 evcnt_detach(&stats->gorc); 792 evcnt_detach(&stats->tpr); 793 evcnt_detach(&stats->gprc); 794 evcnt_detach(&stats->mprc); 795 evcnt_detach(&stats->bprc); 796 evcnt_detach(&stats->prc64); 797 evcnt_detach(&stats->prc127); 798 evcnt_detach(&stats->prc255); 799 evcnt_detach(&stats->prc511); 800 evcnt_detach(&stats->prc1023); 801 evcnt_detach(&stats->prc1522); 802 evcnt_detach(&stats->ruc); 803 evcnt_detach(&stats->rfc); 804 evcnt_detach(&stats->roc); 805 evcnt_detach(&stats->rjc); 806 evcnt_detach(&stats->mngprc); 807 evcnt_detach(&stats->mngptc); 808 evcnt_detach(&stats->xec); 809 810 /* Packet Transmission Stats */ 811 evcnt_detach(&stats->gotc); 812 evcnt_detach(&stats->tpt); 813 evcnt_detach(&stats->gptc); 814 evcnt_detach(&stats->bptc); 815 evcnt_detach(&stats->mptc); 816 evcnt_detach(&stats->mngptc); 817 evcnt_detach(&stats->ptc64); 818 evcnt_detach(&stats->ptc127); 819 evcnt_detach(&stats->ptc255); 820 evcnt_detach(&stats->ptc511); 821 evcnt_detach(&stats->ptc1023); 822 evcnt_detach(&stats->ptc1522); 823 824 /* FC Stats */ 825 evcnt_detach(&stats->fccrc); 826 evcnt_detach(&stats->fclast); 827 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 828 evcnt_detach(&stats->fcoerpdc); 829 evcnt_detach(&stats->fcoeprc); 830 evcnt_detach(&stats->fcoeptc); 831 evcnt_detach(&stats->fcoedwrc); 832 evcnt_detach(&stats->fcoedwtc); 833 } 834 835 ixgbe_free_transmit_structures(adapter); 836 ixgbe_free_receive_structures(adapter); 837 free(adapter->mta, M_DEVBUF); 838 839 IXGBE_CORE_LOCK_DESTROY(adapter); 840 return (0); 841 } 842 843 /********************************************************************* 844 * 845 * Shutdown entry point 846 * 847 **********************************************************************/ 848 849 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 850 static int 851 ixgbe_shutdown(device_t dev) 852 { 853 struct adapter *adapter = device_private(dev); 854 IXGBE_CORE_LOCK(adapter); 855 ixgbe_stop(adapter); 856 IXGBE_CORE_UNLOCK(adapter); 857 return (0); 858 } 859 #endif 860 861 862 /********************************************************************* 863 * Transmit entry point 864 * 865 * ixgbe_start is called by the stack to initiate a transmit. 866 * The driver will remain in this routine as long as there are 867 * packets to transmit and transmit resources are available. 868 * In case resources are not available stack is notified and 869 * the packet is requeued. 870 **********************************************************************/ 871 872 static void 873 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 874 { 875 int rc; 876 struct mbuf *m_head; 877 struct adapter *adapter = txr->adapter; 878 879 IXGBE_TX_LOCK_ASSERT(txr); 880 881 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 882 IFF_RUNNING) 883 return; 884 if (!adapter->link_active) 885 return; 886 887 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 888 889 IFQ_POLL(&ifp->if_snd, m_head); 890 if (m_head == NULL) 891 break; 892 893 if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { 894 ifp->if_flags |= IFF_OACTIVE; 895 break; 896 } 897 IFQ_DEQUEUE(&ifp->if_snd, m_head); 898 if (rc == EFBIG) { 899 struct mbuf *mtmp; 900 901 if ((mtmp = m_defrag(m_head, M_DONTWAIT)) != NULL) { 902 m_head = mtmp; 903 rc = ixgbe_xmit(txr, m_head); 904 if (rc != 0) 905 adapter->efbig2_tx_dma_setup.ev_count++; 906 } else 907 adapter->m_defrag_failed.ev_count++; 908 } 909 if (rc != 0) { 910 m_freem(m_head); 911 continue; 912 } 913 914 /* Send a copy of the frame to the BPF listener */ 915 bpf_mtap(ifp, m_head); 916 917 /* Set watchdog on */ 918 getmicrotime(&txr->watchdog_time); 919 txr->queue_status = IXGBE_QUEUE_WORKING; 920 921 } 922 return; 923 } 924 925 /* 926 * Legacy TX start - called by the stack, this 927 * always uses the first tx ring, and should 928 * not be used with multiqueue tx enabled. 929 */ 930 static void 931 ixgbe_start(struct ifnet *ifp) 932 { 933 struct adapter *adapter = ifp->if_softc; 934 struct tx_ring *txr = adapter->tx_rings; 935 936 if (ifp->if_flags & IFF_RUNNING) { 937 IXGBE_TX_LOCK(txr); 938 ixgbe_start_locked(txr, ifp); 939 IXGBE_TX_UNLOCK(txr); 940 } 941 return; 942 } 943 944 #if __FreeBSD_version >= 800000 945 /* 946 ** Multiqueue Transmit driver 947 ** 948 */ 949 static int 950 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 951 { 952 struct adapter *adapter = ifp->if_softc; 953 struct ix_queue *que; 954 struct tx_ring *txr; 955 int i = 0, err = 0; 956 957 /* Which queue to use */ 958 if ((m->m_flags & M_FLOWID) != 0) 959 i = m->m_pkthdr.flowid % adapter->num_queues; 960 961 txr = &adapter->tx_rings[i]; 962 que = &adapter->queues[i]; 963 964 if (IXGBE_TX_TRYLOCK(txr)) { 965 err = ixgbe_mq_start_locked(ifp, txr, m); 966 IXGBE_TX_UNLOCK(txr); 967 } else { 968 err = drbr_enqueue(ifp, txr->br, m); 969 softint_schedule(que->que_si); 970 } 971 972 return (err); 973 } 974 975 static int 976 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 977 { 978 struct adapter *adapter = txr->adapter; 979 struct mbuf *next; 980 int enqueued, err = 0; 981 982 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 983 IFF_RUNNING || adapter->link_active == 0) { 984 if (m != NULL) 985 err = drbr_enqueue(ifp, txr->br, m); 986 return (err); 987 } 988 989 enqueued = 0; 990 if (m == NULL) { 991 next = drbr_dequeue(ifp, txr->br); 992 } else if (drbr_needs_enqueue(ifp, txr->br)) { 993 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 994 return (err); 995 next = drbr_dequeue(ifp, txr->br); 996 } else 997 next = m; 998 999 /* Process the queue */ 1000 while (next != NULL) { 1001 if ((err = ixgbe_xmit(txr, &next)) != 0) { 1002 if (next != NULL) 1003 err = drbr_enqueue(ifp, txr->br, next); 1004 break; 1005 } 1006 enqueued++; 1007 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 1008 /* Send a copy of the frame to the BPF listener */ 1009 bpf_mtap(ifp, next); 1010 if ((ifp->if_flags & IFF_RUNNING) == 0) 1011 break; 1012 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) 1013 ixgbe_txeof(txr); 1014 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { 1015 ifp->if_flags |= IFF_OACTIVE; 1016 break; 1017 } 1018 next = drbr_dequeue(ifp, txr->br); 1019 } 1020 1021 if (enqueued > 0) { 1022 /* Set watchdog on */ 1023 txr->queue_status = IXGBE_QUEUE_WORKING; 1024 getmicrotime(&txr->watchdog_time); 1025 } 1026 1027 return (err); 1028 } 1029 1030 /* 1031 ** Flush all ring buffers 1032 */ 1033 static void 1034 ixgbe_qflush(struct ifnet *ifp) 1035 { 1036 struct adapter *adapter = ifp->if_softc; 1037 struct tx_ring *txr = adapter->tx_rings; 1038 struct mbuf *m; 1039 1040 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1041 IXGBE_TX_LOCK(txr); 1042 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 1043 m_freem(m); 1044 IXGBE_TX_UNLOCK(txr); 1045 } 1046 if_qflush(ifp); 1047 } 1048 #endif /* __FreeBSD_version >= 800000 */ 1049 1050 static int 1051 ixgbe_ifflags_cb(struct ethercom *ec) 1052 { 1053 struct ifnet *ifp = &ec->ec_if; 1054 struct adapter *adapter = ifp->if_softc; 1055 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 1056 1057 IXGBE_CORE_LOCK(adapter); 1058 1059 if (change != 0) 1060 adapter->if_flags = ifp->if_flags; 1061 1062 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 1063 rc = ENETRESET; 1064 else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0) 1065 ixgbe_set_promisc(adapter); 1066 1067 IXGBE_CORE_UNLOCK(adapter); 1068 1069 return rc; 1070 } 1071 1072 /********************************************************************* 1073 * Ioctl entry point 1074 * 1075 * ixgbe_ioctl is called when the user wants to configure the 1076 * interface. 1077 * 1078 * return 0 on success, positive on failure 1079 **********************************************************************/ 1080 1081 static int 1082 ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data) 1083 { 1084 struct adapter *adapter = ifp->if_softc; 1085 struct ifcapreq *ifcr = data; 1086 struct ifreq *ifr = data; 1087 int error = 0; 1088 int l4csum_en; 1089 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 1090 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 1091 1092 switch (command) { 1093 case SIOCSIFFLAGS: 1094 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1095 break; 1096 case SIOCADDMULTI: 1097 case SIOCDELMULTI: 1098 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 1099 break; 1100 case SIOCSIFMEDIA: 1101 case SIOCGIFMEDIA: 1102 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1103 break; 1104 case SIOCSIFCAP: 1105 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1106 break; 1107 case SIOCSIFMTU: 1108 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1109 break; 1110 default: 1111 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 1112 break; 1113 } 1114 1115 switch (command) { 1116 case SIOCSIFMEDIA: 1117 case SIOCGIFMEDIA: 1118 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1119 case SIOCSIFCAP: 1120 /* Layer-4 Rx checksum offload has to be turned on and 1121 * off as a unit. 1122 */ 1123 l4csum_en = ifcr->ifcr_capenable & l4csum; 1124 if (l4csum_en != l4csum && l4csum_en != 0) 1125 return EINVAL; 1126 /*FALLTHROUGH*/ 1127 case SIOCADDMULTI: 1128 case SIOCDELMULTI: 1129 case SIOCSIFFLAGS: 1130 case SIOCSIFMTU: 1131 default: 1132 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 1133 return error; 1134 if ((ifp->if_flags & IFF_RUNNING) == 0) 1135 ; 1136 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 1137 IXGBE_CORE_LOCK(adapter); 1138 ixgbe_init_locked(adapter); 1139 IXGBE_CORE_UNLOCK(adapter); 1140 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 1141 /* 1142 * Multicast list has changed; set the hardware filter 1143 * accordingly. 1144 */ 1145 IXGBE_CORE_LOCK(adapter); 1146 ixgbe_disable_intr(adapter); 1147 ixgbe_set_multi(adapter); 1148 ixgbe_enable_intr(adapter); 1149 IXGBE_CORE_UNLOCK(adapter); 1150 } 1151 return 0; 1152 } 1153 } 1154 1155 /********************************************************************* 1156 * Init entry point 1157 * 1158 * This routine is used in two ways. It is used by the stack as 1159 * init entry point in network interface structure. It is also used 1160 * by the driver as a hw/sw initialization routine to get to a 1161 * consistent state. 1162 * 1163 * return 0 on success, positive on failure 1164 **********************************************************************/ 1165 #define IXGBE_MHADD_MFS_SHIFT 16 1166 1167 static void 1168 ixgbe_init_locked(struct adapter *adapter) 1169 { 1170 struct ifnet *ifp = adapter->ifp; 1171 device_t dev = adapter->dev; 1172 struct ixgbe_hw *hw = &adapter->hw; 1173 u32 k, txdctl, mhadd, gpie; 1174 u32 rxdctl, rxctrl; 1175 1176 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 1177 1178 KASSERT(mutex_owned(&adapter->core_mtx)); 1179 INIT_DEBUGOUT("ixgbe_init: begin"); 1180 hw->adapter_stopped = FALSE; 1181 ixgbe_stop_adapter(hw); 1182 callout_stop(&adapter->timer); 1183 1184 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 1185 adapter->max_frame_size = 1186 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1187 1188 /* reprogram the RAR[0] in case user changed it. */ 1189 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1190 1191 /* Get the latest mac address, User can use a LAA */ 1192 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 1193 IXGBE_ETH_LENGTH_OF_ADDRESS); 1194 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 1195 hw->addr_ctrl.rar_used_count = 1; 1196 1197 /* Prepare transmit descriptors and buffers */ 1198 if (ixgbe_setup_transmit_structures(adapter)) { 1199 device_printf(dev,"Could not setup transmit structures\n"); 1200 ixgbe_stop(adapter); 1201 return; 1202 } 1203 1204 ixgbe_init_hw(hw); 1205 ixgbe_initialize_transmit_units(adapter); 1206 1207 /* Setup Multicast table */ 1208 ixgbe_set_multi(adapter); 1209 1210 /* 1211 ** Determine the correct mbuf pool 1212 ** for doing jumbo/headersplit 1213 */ 1214 if (adapter->max_frame_size <= 2048) 1215 adapter->rx_mbuf_sz = MCLBYTES; 1216 else if (adapter->max_frame_size <= 4096) 1217 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1218 else if (adapter->max_frame_size <= 9216) 1219 adapter->rx_mbuf_sz = MJUM9BYTES; 1220 else 1221 adapter->rx_mbuf_sz = MJUM16BYTES; 1222 1223 /* Prepare receive descriptors and buffers */ 1224 if (ixgbe_setup_receive_structures(adapter)) { 1225 device_printf(dev,"Could not setup receive structures\n"); 1226 ixgbe_stop(adapter); 1227 return; 1228 } 1229 1230 /* Configure RX settings */ 1231 ixgbe_initialize_receive_units(adapter); 1232 1233 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 1234 1235 /* Enable Fan Failure Interrupt */ 1236 gpie |= IXGBE_SDP1_GPIEN; 1237 1238 /* Add for Thermal detection */ 1239 if (hw->mac.type == ixgbe_mac_82599EB) 1240 gpie |= IXGBE_SDP2_GPIEN; 1241 1242 if (adapter->msix > 1) { 1243 /* Enable Enhanced MSIX mode */ 1244 gpie |= IXGBE_GPIE_MSIX_MODE; 1245 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1246 IXGBE_GPIE_OCD; 1247 } 1248 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 1249 1250 /* Set MTU size */ 1251 if (ifp->if_mtu > ETHERMTU) { 1252 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 1253 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1254 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1255 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 1256 } 1257 1258 /* Now enable all the queues */ 1259 1260 for (int i = 0; i < adapter->num_queues; i++) { 1261 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1262 txdctl |= IXGBE_TXDCTL_ENABLE; 1263 /* Set WTHRESH to 8, burst writeback */ 1264 txdctl |= (8 << 16); 1265 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1266 } 1267 1268 for (int i = 0; i < adapter->num_queues; i++) { 1269 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1270 if (hw->mac.type == ixgbe_mac_82598EB) { 1271 /* 1272 ** PTHRESH = 21 1273 ** HTHRESH = 4 1274 ** WTHRESH = 8 1275 */ 1276 rxdctl &= ~0x3FFFFF; 1277 rxdctl |= 0x080420; 1278 } 1279 rxdctl |= IXGBE_RXDCTL_ENABLE; 1280 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1281 /* XXX I don't trust this loop, and I don't trust the 1282 * XXX memory barrier. What is this meant to do? --dyoung 1283 */ 1284 for (k = 0; k < 10; k++) { 1285 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1286 IXGBE_RXDCTL_ENABLE) 1287 break; 1288 else 1289 msec_delay(1); 1290 } 1291 wmb(); 1292 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); 1293 } 1294 1295 /* Set up VLAN support and filter */ 1296 ixgbe_setup_vlan_hw_support(adapter); 1297 1298 /* Enable Receive engine */ 1299 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1300 if (hw->mac.type == ixgbe_mac_82598EB) 1301 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1302 rxctrl |= IXGBE_RXCTRL_RXEN; 1303 ixgbe_enable_rx_dma(hw, rxctrl); 1304 1305 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1306 1307 /* Set up MSI/X routing */ 1308 if (ixgbe_enable_msix) { 1309 ixgbe_configure_ivars(adapter); 1310 /* Set up auto-mask */ 1311 if (hw->mac.type == ixgbe_mac_82598EB) 1312 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1313 else { 1314 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1315 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1316 } 1317 } else { /* Simple settings for Legacy/MSI */ 1318 ixgbe_set_ivar(adapter, 0, 0, 0); 1319 ixgbe_set_ivar(adapter, 0, 0, 1); 1320 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1321 } 1322 1323 #ifdef IXGBE_FDIR 1324 /* Init Flow director */ 1325 if (hw->mac.type != ixgbe_mac_82598EB) 1326 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); 1327 #endif 1328 1329 /* 1330 ** Check on any SFP devices that 1331 ** need to be kick-started 1332 */ 1333 if (hw->phy.type == ixgbe_phy_none) { 1334 int err = hw->phy.ops.identify(hw); 1335 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1336 device_printf(dev, 1337 "Unsupported SFP+ module type was detected.\n"); 1338 return; 1339 } 1340 } 1341 1342 /* Set moderation on the Link interrupt */ 1343 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 1344 1345 /* Config/Enable Link */ 1346 ixgbe_config_link(adapter); 1347 1348 /* And now turn on interrupts */ 1349 ixgbe_enable_intr(adapter); 1350 1351 /* Now inform the stack we're ready */ 1352 ifp->if_flags |= IFF_RUNNING; 1353 ifp->if_flags &= ~IFF_OACTIVE; 1354 1355 return; 1356 } 1357 1358 static int 1359 ixgbe_init(struct ifnet *ifp) 1360 { 1361 struct adapter *adapter = ifp->if_softc; 1362 1363 IXGBE_CORE_LOCK(adapter); 1364 ixgbe_init_locked(adapter); 1365 IXGBE_CORE_UNLOCK(adapter); 1366 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 1367 } 1368 1369 1370 /* 1371 ** 1372 ** MSIX Interrupt Handlers and Tasklets 1373 ** 1374 */ 1375 1376 static inline void 1377 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 1378 { 1379 struct ixgbe_hw *hw = &adapter->hw; 1380 u64 queue = (u64)(1 << vector); 1381 u32 mask; 1382 1383 if (hw->mac.type == ixgbe_mac_82598EB) { 1384 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1385 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 1386 } else { 1387 mask = (queue & 0xFFFFFFFF); 1388 if (mask) 1389 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 1390 mask = (queue >> 32); 1391 if (mask) 1392 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 1393 } 1394 } 1395 1396 static inline void 1397 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 1398 { 1399 struct ixgbe_hw *hw = &adapter->hw; 1400 u64 queue = (u64)(1 << vector); 1401 u32 mask; 1402 1403 if (hw->mac.type == ixgbe_mac_82598EB) { 1404 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1405 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 1406 } else { 1407 mask = (queue & 0xFFFFFFFF); 1408 if (mask) 1409 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 1410 mask = (queue >> 32); 1411 if (mask) 1412 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 1413 } 1414 } 1415 1416 static inline void 1417 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 1418 { 1419 u32 mask; 1420 1421 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 1422 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1423 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 1424 } else { 1425 mask = (queues & 0xFFFFFFFF); 1426 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 1427 mask = (queues >> 32); 1428 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 1429 } 1430 } 1431 1432 1433 static void 1434 ixgbe_handle_que(void *context) 1435 { 1436 struct ix_queue *que = context; 1437 struct adapter *adapter = que->adapter; 1438 struct tx_ring *txr = que->txr; 1439 struct ifnet *ifp = adapter->ifp; 1440 bool more; 1441 1442 adapter->handleq.ev_count++; 1443 1444 if (ifp->if_flags & IFF_RUNNING) { 1445 more = ixgbe_rxeof(que, adapter->rx_process_limit); 1446 IXGBE_TX_LOCK(txr); 1447 ixgbe_txeof(txr); 1448 #if __FreeBSD_version >= 800000 1449 if (!drbr_empty(ifp, txr->br)) 1450 ixgbe_mq_start_locked(ifp, txr, NULL); 1451 #else 1452 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1453 ixgbe_start_locked(txr, ifp); 1454 #endif 1455 IXGBE_TX_UNLOCK(txr); 1456 if (more) { 1457 adapter->req.ev_count++; 1458 softint_schedule(que->que_si); 1459 return; 1460 } 1461 } 1462 1463 /* Reenable this interrupt */ 1464 ixgbe_enable_queue(adapter, que->msix); 1465 1466 return; 1467 } 1468 1469 1470 /********************************************************************* 1471 * 1472 * Legacy Interrupt Service routine 1473 * 1474 **********************************************************************/ 1475 1476 static int 1477 ixgbe_legacy_irq(void *arg) 1478 { 1479 struct ix_queue *que = arg; 1480 struct adapter *adapter = que->adapter; 1481 struct ixgbe_hw *hw = &adapter->hw; 1482 struct tx_ring *txr = adapter->tx_rings; 1483 bool more_tx, more_rx; 1484 u32 reg_eicr, loop = MAX_LOOP; 1485 1486 1487 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1488 1489 adapter->stats.legint.ev_count++; 1490 ++que->irqs; 1491 if (reg_eicr == 0) { 1492 adapter->stats.intzero.ev_count++; 1493 ixgbe_enable_intr(adapter); 1494 return 0; 1495 } 1496 1497 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1498 1499 IXGBE_TX_LOCK(txr); 1500 do { 1501 adapter->txloops.ev_count++; 1502 more_tx = ixgbe_txeof(txr); 1503 } while (loop-- && more_tx); 1504 IXGBE_TX_UNLOCK(txr); 1505 1506 if (more_rx || more_tx) { 1507 if (more_rx) 1508 adapter->morerx.ev_count++; 1509 if (more_tx) 1510 adapter->moretx.ev_count++; 1511 softint_schedule(que->que_si); 1512 } 1513 1514 /* Check for fan failure */ 1515 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1516 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1517 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1518 "REPLACE IMMEDIATELY!!\n"); 1519 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1520 } 1521 1522 /* Link status change */ 1523 if (reg_eicr & IXGBE_EICR_LSC) 1524 softint_schedule(adapter->link_si); 1525 1526 ixgbe_enable_intr(adapter); 1527 return 1; 1528 } 1529 1530 1531 #if defined(NETBSD_MSI_OR_MSIX) 1532 /********************************************************************* 1533 * 1534 * MSI Queue Interrupt Service routine 1535 * 1536 **********************************************************************/ 1537 void 1538 ixgbe_msix_que(void *arg) 1539 { 1540 struct ix_queue *que = arg; 1541 struct adapter *adapter = que->adapter; 1542 struct tx_ring *txr = que->txr; 1543 struct rx_ring *rxr = que->rxr; 1544 bool more_tx, more_rx; 1545 u32 newitr = 0; 1546 1547 ++que->irqs; 1548 1549 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1550 1551 IXGBE_TX_LOCK(txr); 1552 more_tx = ixgbe_txeof(txr); 1553 IXGBE_TX_UNLOCK(txr); 1554 1555 /* Do AIM now? */ 1556 1557 if (ixgbe_enable_aim == FALSE) 1558 goto no_calc; 1559 /* 1560 ** Do Adaptive Interrupt Moderation: 1561 ** - Write out last calculated setting 1562 ** - Calculate based on average size over 1563 ** the last interval. 1564 */ 1565 if (que->eitr_setting) 1566 IXGBE_WRITE_REG(&adapter->hw, 1567 IXGBE_EITR(que->msix), que->eitr_setting); 1568 1569 que->eitr_setting = 0; 1570 1571 /* Idle, do nothing */ 1572 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1573 goto no_calc; 1574 1575 if ((txr->bytes) && (txr->packets)) 1576 newitr = txr->bytes/txr->packets; 1577 if ((rxr->bytes) && (rxr->packets)) 1578 newitr = max(newitr, 1579 (rxr->bytes / rxr->packets)); 1580 newitr += 24; /* account for hardware frame, crc */ 1581 1582 /* set an upper boundary */ 1583 newitr = min(newitr, 3000); 1584 1585 /* Be nice to the mid range */ 1586 if ((newitr > 300) && (newitr < 1200)) 1587 newitr = (newitr / 3); 1588 else 1589 newitr = (newitr / 2); 1590 1591 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1592 newitr |= newitr << 16; 1593 else 1594 newitr |= IXGBE_EITR_CNT_WDIS; 1595 1596 /* save for next interrupt */ 1597 que->eitr_setting = newitr; 1598 1599 /* Reset state */ 1600 txr->bytes = 0; 1601 txr->packets = 0; 1602 rxr->bytes = 0; 1603 rxr->packets = 0; 1604 1605 no_calc: 1606 if (more_tx || more_rx) 1607 softint_schedule(que->que_si); 1608 else /* Reenable this interrupt */ 1609 ixgbe_enable_queue(adapter, que->msix); 1610 return; 1611 } 1612 1613 1614 static void 1615 ixgbe_msix_link(void *arg) 1616 { 1617 struct adapter *adapter = arg; 1618 struct ixgbe_hw *hw = &adapter->hw; 1619 u32 reg_eicr; 1620 1621 ++adapter->link_irq.ev_count; 1622 1623 /* First get the cause */ 1624 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1625 /* Clear interrupt with write */ 1626 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1627 1628 /* Link status change */ 1629 if (reg_eicr & IXGBE_EICR_LSC) 1630 softint_schedule(adapter->link_si); 1631 1632 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 1633 #ifdef IXGBE_FDIR 1634 if (reg_eicr & IXGBE_EICR_FLOW_DIR) { 1635 /* This is probably overkill :) */ 1636 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) 1637 return; 1638 /* Clear the interrupt */ 1639 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR); 1640 /* Turn off the interface */ 1641 adapter->ifp->if_flags &= ~IFF_RUNNING; 1642 softint_schedule(adapter->fdir_si); 1643 } else 1644 #endif 1645 if (reg_eicr & IXGBE_EICR_ECC) { 1646 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " 1647 "Please Reboot!!\n"); 1648 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1649 } else 1650 1651 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1652 /* Clear the interrupt */ 1653 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1654 softint_schedule(adapter->msf_si); 1655 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1656 /* Clear the interrupt */ 1657 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1658 softint_schedule(adapter->mod_si); 1659 } 1660 } 1661 1662 /* Check for fan failure */ 1663 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1664 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1665 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1666 "REPLACE IMMEDIATELY!!\n"); 1667 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1668 } 1669 1670 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1671 return; 1672 } 1673 #endif 1674 1675 /********************************************************************* 1676 * 1677 * Media Ioctl callback 1678 * 1679 * This routine is called whenever the user queries the status of 1680 * the interface using ifconfig. 1681 * 1682 **********************************************************************/ 1683 static void 1684 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1685 { 1686 struct adapter *adapter = ifp->if_softc; 1687 1688 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1689 IXGBE_CORE_LOCK(adapter); 1690 ixgbe_update_link_status(adapter); 1691 1692 ifmr->ifm_status = IFM_AVALID; 1693 ifmr->ifm_active = IFM_ETHER; 1694 1695 if (!adapter->link_active) { 1696 IXGBE_CORE_UNLOCK(adapter); 1697 return; 1698 } 1699 1700 ifmr->ifm_status |= IFM_ACTIVE; 1701 1702 switch (adapter->link_speed) { 1703 case IXGBE_LINK_SPEED_1GB_FULL: 1704 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1705 break; 1706 case IXGBE_LINK_SPEED_10GB_FULL: 1707 ifmr->ifm_active |= adapter->optics | IFM_FDX; 1708 break; 1709 } 1710 1711 IXGBE_CORE_UNLOCK(adapter); 1712 1713 return; 1714 } 1715 1716 /********************************************************************* 1717 * 1718 * Media Ioctl callback 1719 * 1720 * This routine is called when the user changes speed/duplex using 1721 * media/mediopt option with ifconfig. 1722 * 1723 **********************************************************************/ 1724 static int 1725 ixgbe_media_change(struct ifnet * ifp) 1726 { 1727 struct adapter *adapter = ifp->if_softc; 1728 struct ifmedia *ifm = &adapter->media; 1729 1730 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1731 1732 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1733 return (EINVAL); 1734 1735 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1736 case IFM_AUTO: 1737 adapter->hw.phy.autoneg_advertised = 1738 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 1739 break; 1740 default: 1741 device_printf(adapter->dev, "Only auto media type\n"); 1742 return (EINVAL); 1743 } 1744 1745 return (0); 1746 } 1747 1748 /********************************************************************* 1749 * 1750 * This routine maps the mbufs to tx descriptors, allowing the 1751 * TX engine to transmit the packets. 1752 * - return 0 on success, positive on failure 1753 * 1754 **********************************************************************/ 1755 1756 static int 1757 ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) 1758 { 1759 struct m_tag *mtag; 1760 struct adapter *adapter = txr->adapter; 1761 struct ethercom *ec = &adapter->osdep.ec; 1762 u32 olinfo_status = 0, cmd_type_len; 1763 u32 paylen = 0; 1764 int i, j, error; 1765 int first, last = 0; 1766 bus_dmamap_t map; 1767 struct ixgbe_tx_buf *txbuf, *txbuf_mapped; 1768 union ixgbe_adv_tx_desc *txd = NULL; 1769 1770 /* Basic descriptor defines */ 1771 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1772 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1773 1774 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1775 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1776 1777 /* 1778 * Important to capture the first descriptor 1779 * used because it will contain the index of 1780 * the one we tell the hardware to report back 1781 */ 1782 first = txr->next_avail_desc; 1783 txbuf = &txr->tx_buffers[first]; 1784 txbuf_mapped = txbuf; 1785 map = txbuf->map; 1786 1787 /* 1788 * Map the packet for DMA. 1789 */ 1790 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1791 m_head, BUS_DMA_NOWAIT); 1792 1793 switch (error) { 1794 case EAGAIN: 1795 adapter->eagain_tx_dma_setup.ev_count++; 1796 return EAGAIN; 1797 case ENOMEM: 1798 adapter->enomem_tx_dma_setup.ev_count++; 1799 return EAGAIN; 1800 case EFBIG: 1801 adapter->efbig_tx_dma_setup.ev_count++; 1802 return error; 1803 case EINVAL: 1804 adapter->einval_tx_dma_setup.ev_count++; 1805 return error; 1806 default: 1807 adapter->other_tx_dma_setup.ev_count++; 1808 return error; 1809 case 0: 1810 break; 1811 } 1812 1813 /* Make certain there are enough descriptors */ 1814 if (map->dm_nsegs > txr->tx_avail - 2) { 1815 txr->no_desc_avail.ev_count++; 1816 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1817 return EAGAIN; 1818 } 1819 1820 /* 1821 ** Set up the appropriate offload context 1822 ** this becomes the first descriptor of 1823 ** a packet. 1824 */ 1825 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1826 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 1827 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1828 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1829 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1830 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1831 ++adapter->tso_tx.ev_count; 1832 } else { 1833 ++adapter->tso_err.ev_count; 1834 /* XXX unload DMA map! --dyoung */ 1835 return ENXIO; 1836 } 1837 } else 1838 olinfo_status |= ixgbe_tx_ctx_setup(txr, m_head); 1839 1840 #ifdef IXGBE_IEEE1588 1841 /* This is changing soon to an mtag detection */ 1842 if (we detect this mbuf has a TSTAMP mtag) 1843 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; 1844 #endif 1845 1846 #ifdef IXGBE_FDIR 1847 /* Do the flow director magic */ 1848 if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 1849 ++txr->atr_count; 1850 if (txr->atr_count >= atr_sample_rate) { 1851 ixgbe_atr(txr, m_head); 1852 txr->atr_count = 0; 1853 } 1854 } 1855 #endif 1856 /* Record payload length */ 1857 if (paylen == 0) 1858 olinfo_status |= m_head->m_pkthdr.len << 1859 IXGBE_ADVTXD_PAYLEN_SHIFT; 1860 1861 i = txr->next_avail_desc; 1862 for (j = 0; j < map->dm_nsegs; j++) { 1863 bus_size_t seglen; 1864 bus_addr_t segaddr; 1865 1866 txbuf = &txr->tx_buffers[i]; 1867 txd = &txr->tx_base[i]; 1868 seglen = map->dm_segs[j].ds_len; 1869 segaddr = htole64(map->dm_segs[j].ds_addr); 1870 1871 txd->read.buffer_addr = segaddr; 1872 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1873 cmd_type_len |seglen); 1874 txd->read.olinfo_status = htole32(olinfo_status); 1875 last = i; /* descriptor that will get completion IRQ */ 1876 1877 if (++i == adapter->num_tx_desc) 1878 i = 0; 1879 1880 txbuf->m_head = NULL; 1881 txbuf->eop_index = -1; 1882 } 1883 1884 txd->read.cmd_type_len |= 1885 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1886 txr->tx_avail -= map->dm_nsegs; 1887 txr->next_avail_desc = i; 1888 1889 txbuf->m_head = m_head; 1890 /* We exchange the maps instead of copying because otherwise 1891 * we end up with many pointers to the same map and we free 1892 * one map twice in ixgbe_free_transmit_structures(). Who 1893 * knows what other problems this caused. --dyoung 1894 */ 1895 txr->tx_buffers[first].map = txbuf->map; 1896 txbuf->map = map; 1897 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1898 BUS_DMASYNC_PREWRITE); 1899 1900 /* Set the index of the descriptor that will be marked done */ 1901 txbuf = &txr->tx_buffers[first]; 1902 txbuf->eop_index = last; 1903 1904 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1905 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1906 /* 1907 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1908 * hardware that this frame is available to transmit. 1909 */ 1910 ++txr->total_packets.ev_count; 1911 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); 1912 1913 return 0; 1914 } 1915 1916 static void 1917 ixgbe_set_promisc(struct adapter *adapter) 1918 { 1919 u_int32_t reg_rctl; 1920 struct ifnet *ifp = adapter->ifp; 1921 1922 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1923 reg_rctl &= (~IXGBE_FCTRL_UPE); 1924 reg_rctl &= (~IXGBE_FCTRL_MPE); 1925 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1926 1927 if (ifp->if_flags & IFF_PROMISC) { 1928 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1929 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1930 } else if (ifp->if_flags & IFF_ALLMULTI) { 1931 reg_rctl |= IXGBE_FCTRL_MPE; 1932 reg_rctl &= ~IXGBE_FCTRL_UPE; 1933 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1934 } 1935 return; 1936 } 1937 1938 1939 /********************************************************************* 1940 * Multicast Update 1941 * 1942 * This routine is called whenever multicast address list is updated. 1943 * 1944 **********************************************************************/ 1945 #define IXGBE_RAR_ENTRIES 16 1946 1947 static void 1948 ixgbe_set_multi(struct adapter *adapter) 1949 { 1950 struct ether_multi *enm; 1951 struct ether_multistep step; 1952 u32 fctrl; 1953 u8 *mta; 1954 u8 *update_ptr; 1955 int mcnt = 0; 1956 struct ethercom *ec = &adapter->osdep.ec; 1957 struct ifnet *ifp = adapter->ifp; 1958 1959 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1960 1961 mta = adapter->mta; 1962 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1963 MAX_NUM_MULTICAST_ADDRESSES); 1964 1965 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1966 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1967 if (ifp->if_flags & IFF_PROMISC) 1968 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1969 else if (ifp->if_flags & IFF_ALLMULTI) { 1970 fctrl |= IXGBE_FCTRL_MPE; 1971 fctrl &= ~IXGBE_FCTRL_UPE; 1972 } else 1973 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1974 1975 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1976 1977 ETHER_FIRST_MULTI(step, ec, enm); 1978 while (enm != NULL) { 1979 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1980 ETHER_ADDR_LEN) != 0) { 1981 fctrl |= IXGBE_FCTRL_MPE; 1982 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1983 break; 1984 } 1985 bcopy(enm->enm_addrlo, 1986 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1987 IXGBE_ETH_LENGTH_OF_ADDRESS); 1988 mcnt++; 1989 ETHER_NEXT_MULTI(step, enm); 1990 } 1991 1992 update_ptr = mta; 1993 ixgbe_update_mc_addr_list(&adapter->hw, 1994 update_ptr, mcnt, ixgbe_mc_array_itr); 1995 1996 return; 1997 } 1998 1999 /* 2000 * This is an iterator function now needed by the multicast 2001 * shared code. It simply feeds the shared code routine the 2002 * addresses in the array of ixgbe_set_multi() one by one. 2003 */ 2004 static u8 * 2005 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 2006 { 2007 u8 *addr = *update_ptr; 2008 u8 *newptr; 2009 *vmdq = 0; 2010 2011 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 2012 *update_ptr = newptr; 2013 return addr; 2014 } 2015 2016 2017 /********************************************************************* 2018 * Timer routine 2019 * 2020 * This routine checks for link status,updates statistics, 2021 * and runs the watchdog check. 2022 * 2023 **********************************************************************/ 2024 2025 static void 2026 ixgbe_local_timer1(void *arg) 2027 { 2028 struct adapter *adapter = arg; 2029 device_t dev = adapter->dev; 2030 struct tx_ring *txr = adapter->tx_rings; 2031 2032 KASSERT(mutex_owned(&adapter->core_mtx)); 2033 2034 /* Check for pluggable optics */ 2035 if (adapter->sfp_probe) 2036 if (!ixgbe_sfp_probe(adapter)) 2037 goto out; /* Nothing to do */ 2038 2039 ixgbe_update_link_status(adapter); 2040 ixgbe_update_stats_counters(adapter); 2041 2042 /* 2043 * If the interface has been paused 2044 * then don't do the watchdog check 2045 */ 2046 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 2047 goto out; 2048 2049 /* 2050 ** Check status on the TX queues for a hang 2051 */ 2052 for (int i = 0; i < adapter->num_queues; i++, txr++) 2053 if (txr->queue_status == IXGBE_QUEUE_HUNG) 2054 goto hung; 2055 2056 out: 2057 ixgbe_rearm_queues(adapter, adapter->que_mask); 2058 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 2059 return; 2060 2061 hung: 2062 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2063 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2064 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), 2065 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); 2066 device_printf(dev,"TX(%d) desc avail = %d," 2067 "Next TX to Clean = %d\n", 2068 txr->me, txr->tx_avail, txr->next_to_clean); 2069 adapter->ifp->if_flags &= ~IFF_RUNNING; 2070 adapter->watchdog_events.ev_count++; 2071 ixgbe_init_locked(adapter); 2072 } 2073 2074 static void 2075 ixgbe_local_timer(void *arg) 2076 { 2077 struct adapter *adapter = arg; 2078 2079 IXGBE_CORE_LOCK(adapter); 2080 ixgbe_local_timer1(adapter); 2081 IXGBE_CORE_UNLOCK(adapter); 2082 } 2083 2084 /* 2085 ** Note: this routine updates the OS on the link state 2086 ** the real check of the hardware only happens with 2087 ** a link interrupt. 2088 */ 2089 static void 2090 ixgbe_update_link_status(struct adapter *adapter) 2091 { 2092 struct ifnet *ifp = adapter->ifp; 2093 struct tx_ring *txr = adapter->tx_rings; 2094 device_t dev = adapter->dev; 2095 2096 2097 if (adapter->link_up){ 2098 if (adapter->link_active == FALSE) { 2099 if (bootverbose) 2100 device_printf(dev,"Link is up %d Gbps %s \n", 2101 ((adapter->link_speed == 128)? 10:1), 2102 "Full Duplex"); 2103 adapter->link_active = TRUE; 2104 if_link_state_change(ifp, LINK_STATE_UP); 2105 } 2106 } else { /* Link down */ 2107 if (adapter->link_active == TRUE) { 2108 if (bootverbose) 2109 device_printf(dev,"Link is Down\n"); 2110 if_link_state_change(ifp, LINK_STATE_DOWN); 2111 adapter->link_active = FALSE; 2112 for (int i = 0; i < adapter->num_queues; 2113 i++, txr++) 2114 txr->queue_status = IXGBE_QUEUE_IDLE; 2115 } 2116 } 2117 2118 return; 2119 } 2120 2121 2122 static void 2123 ixgbe_ifstop(struct ifnet *ifp, int disable) 2124 { 2125 struct adapter *adapter = ifp->if_softc; 2126 2127 IXGBE_CORE_LOCK(adapter); 2128 ixgbe_stop(adapter); 2129 IXGBE_CORE_UNLOCK(adapter); 2130 } 2131 2132 /********************************************************************* 2133 * 2134 * This routine disables all traffic on the adapter by issuing a 2135 * global reset on the MAC and deallocates TX/RX buffers. 2136 * 2137 **********************************************************************/ 2138 2139 static void 2140 ixgbe_stop(void *arg) 2141 { 2142 struct ifnet *ifp; 2143 struct adapter *adapter = arg; 2144 struct ixgbe_hw *hw = &adapter->hw; 2145 ifp = adapter->ifp; 2146 2147 KASSERT(mutex_owned(&adapter->core_mtx)); 2148 2149 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 2150 ixgbe_disable_intr(adapter); 2151 2152 /* Tell the stack that the interface is no longer active */ 2153 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 2154 2155 ixgbe_reset_hw(hw); 2156 hw->adapter_stopped = FALSE; 2157 ixgbe_stop_adapter(hw); 2158 /* Turn off the laser */ 2159 if (hw->phy.multispeed_fiber) 2160 ixgbe_disable_tx_laser(hw); 2161 callout_stop(&adapter->timer); 2162 2163 /* reprogram the RAR[0] in case user changed it. */ 2164 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 2165 2166 return; 2167 } 2168 2169 2170 /********************************************************************* 2171 * 2172 * Determine hardware revision. 2173 * 2174 **********************************************************************/ 2175 static void 2176 ixgbe_identify_hardware(struct adapter *adapter) 2177 { 2178 pcitag_t tag; 2179 pci_chipset_tag_t pc; 2180 pcireg_t subid, id; 2181 struct ixgbe_hw *hw = &adapter->hw; 2182 2183 pc = adapter->osdep.pc; 2184 tag = adapter->osdep.tag; 2185 2186 id = pci_conf_read(pc, tag, PCI_ID_REG); 2187 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 2188 2189 /* Save off the information about this board */ 2190 hw->vendor_id = PCI_VENDOR(id); 2191 hw->device_id = PCI_PRODUCT(id); 2192 hw->revision_id = 2193 PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 2194 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 2195 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 2196 2197 /* We need this here to set the num_segs below */ 2198 ixgbe_set_mac_type(hw); 2199 2200 /* Pick up the 82599 and VF settings */ 2201 if (hw->mac.type != ixgbe_mac_82598EB) { 2202 hw->phy.smart_speed = ixgbe_smart_speed; 2203 adapter->num_segs = IXGBE_82599_SCATTER; 2204 } else 2205 adapter->num_segs = IXGBE_82598_SCATTER; 2206 2207 return; 2208 } 2209 2210 /********************************************************************* 2211 * 2212 * Determine optic type 2213 * 2214 **********************************************************************/ 2215 static void 2216 ixgbe_setup_optics(struct adapter *adapter) 2217 { 2218 struct ixgbe_hw *hw = &adapter->hw; 2219 int layer; 2220 2221 layer = ixgbe_get_supported_physical_layer(hw); 2222 switch (layer) { 2223 case IXGBE_PHYSICAL_LAYER_10GBASE_T: 2224 adapter->optics = IFM_10G_T; 2225 break; 2226 case IXGBE_PHYSICAL_LAYER_1000BASE_T: 2227 adapter->optics = IFM_1000_T; 2228 break; 2229 case IXGBE_PHYSICAL_LAYER_10GBASE_LR: 2230 case IXGBE_PHYSICAL_LAYER_10GBASE_LRM: 2231 adapter->optics = IFM_10G_LR; 2232 break; 2233 case IXGBE_PHYSICAL_LAYER_10GBASE_SR: 2234 adapter->optics = IFM_10G_SR; 2235 break; 2236 case IXGBE_PHYSICAL_LAYER_10GBASE_KX4: 2237 case IXGBE_PHYSICAL_LAYER_10GBASE_CX4: 2238 adapter->optics = IFM_10G_CX4; 2239 break; 2240 case IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU: 2241 adapter->optics = IFM_10G_TWINAX; 2242 break; 2243 case IXGBE_PHYSICAL_LAYER_1000BASE_KX: 2244 case IXGBE_PHYSICAL_LAYER_10GBASE_KR: 2245 case IXGBE_PHYSICAL_LAYER_10GBASE_XAUI: 2246 case IXGBE_PHYSICAL_LAYER_UNKNOWN: 2247 default: 2248 adapter->optics = IFM_ETHER | IFM_AUTO; 2249 break; 2250 } 2251 return; 2252 } 2253 2254 /********************************************************************* 2255 * 2256 * Setup the Legacy or MSI Interrupt handler 2257 * 2258 **********************************************************************/ 2259 static int 2260 ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) 2261 { 2262 device_t dev = adapter->dev; 2263 struct ix_queue *que = adapter->queues; 2264 int rid = 0; 2265 2266 /* MSI RID at 1 */ 2267 if (adapter->msix == 1) 2268 rid = 1; 2269 2270 /* We allocate a single interrupt resource */ 2271 if (pci_intr_map(pa, &adapter->osdep.ih) != 0) { 2272 aprint_error_dev(dev, "unable to map interrupt\n"); 2273 return ENXIO; 2274 } else { 2275 aprint_normal_dev(dev, "interrupting at %s\n", 2276 pci_intr_string(adapter->osdep.pc, adapter->osdep.ih)); 2277 } 2278 2279 /* 2280 * Try allocating a fast interrupt and the associated deferred 2281 * processing contexts. 2282 */ 2283 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); 2284 2285 /* Tasklets for Link, SFP and Multispeed Fiber */ 2286 adapter->link_si = 2287 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2288 adapter->mod_si = 2289 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2290 adapter->msf_si = 2291 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2292 2293 #ifdef IXGBE_FDIR 2294 adapter->fdir_si = 2295 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2296 #endif 2297 if (que->que_si == NULL || 2298 adapter->link_si == NULL || 2299 adapter->mod_si == NULL || 2300 #ifdef IXGBE_FDIR 2301 adapter->fdir_si == NULL || 2302 #endif 2303 adapter->msf_si == NULL) { 2304 aprint_error_dev(dev, 2305 "could not establish software interrupts\n"); 2306 return ENXIO; 2307 } 2308 2309 adapter->osdep.intr = pci_intr_establish(adapter->osdep.pc, 2310 adapter->osdep.ih, IPL_NET, ixgbe_legacy_irq, que); 2311 if (adapter->osdep.intr == NULL) { 2312 aprint_error_dev(dev, "failed to register interrupt handler\n"); 2313 softint_disestablish(que->que_si); 2314 softint_disestablish(adapter->link_si); 2315 softint_disestablish(adapter->mod_si); 2316 softint_disestablish(adapter->msf_si); 2317 #ifdef IXGBE_FDIR 2318 softint_disestablish(adapter->fdir_si); 2319 #endif 2320 return ENXIO; 2321 } 2322 /* For simplicity in the handlers */ 2323 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK; 2324 2325 return (0); 2326 } 2327 2328 2329 /********************************************************************* 2330 * 2331 * Setup MSIX Interrupt resources and handlers 2332 * 2333 **********************************************************************/ 2334 static int 2335 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2336 { 2337 #if !defined(NETBSD_MSI_OR_MSIX) 2338 return 0; 2339 #else 2340 device_t dev = adapter->dev; 2341 struct ix_queue *que = adapter->queues; 2342 int error, rid, vector = 0; 2343 2344 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2345 rid = vector + 1; 2346 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2347 RF_SHAREABLE | RF_ACTIVE); 2348 if (que->res == NULL) { 2349 aprint_error_dev(dev,"Unable to allocate" 2350 " bus resource: que interrupt [%d]\n", vector); 2351 return (ENXIO); 2352 } 2353 /* Set the handler function */ 2354 error = bus_setup_intr(dev, que->res, 2355 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2356 ixgbe_msix_que, que, &que->tag); 2357 if (error) { 2358 que->res = NULL; 2359 aprint_error_dev(dev, 2360 "Failed to register QUE handler\n"); 2361 return error; 2362 } 2363 #if __FreeBSD_version >= 800504 2364 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2365 #endif 2366 que->msix = vector; 2367 adapter->que_mask |= (u64)(1 << que->msix); 2368 /* 2369 ** Bind the msix vector, and thus the 2370 ** ring to the corresponding cpu. 2371 */ 2372 if (adapter->num_queues > 1) 2373 bus_bind_intr(dev, que->res, i); 2374 2375 que->que_si = softint_establish(ixgbe_handle_que, que); 2376 if (que->que_si == NULL) { 2377 aprint_error_dev(dev, 2378 "could not establish software interrupt\n"); 2379 } 2380 } 2381 2382 /* and Link */ 2383 rid = vector + 1; 2384 adapter->res = bus_alloc_resource_any(dev, 2385 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2386 if (!adapter->res) { 2387 aprint_error_dev(dev,"Unable to allocate bus resource: " 2388 "Link interrupt [%d]\n", rid); 2389 return (ENXIO); 2390 } 2391 /* Set the link handler function */ 2392 error = bus_setup_intr(dev, adapter->res, 2393 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2394 ixgbe_msix_link, adapter, &adapter->tag); 2395 if (error) { 2396 adapter->res = NULL; 2397 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2398 return (error); 2399 } 2400 #if __FreeBSD_version >= 800504 2401 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2402 #endif 2403 adapter->linkvec = vector; 2404 /* Tasklets for Link, SFP and Multispeed Fiber */ 2405 adapter->link_si = 2406 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2407 adapter->mod_si = 2408 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2409 adapter->msf_si = 2410 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2411 #ifdef IXGBE_FDIR 2412 adapter->fdir_si = 2413 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2414 #endif 2415 2416 return (0); 2417 #endif 2418 } 2419 2420 /* 2421 * Setup Either MSI/X or MSI 2422 */ 2423 static int 2424 ixgbe_setup_msix(struct adapter *adapter) 2425 { 2426 #if !defined(NETBSD_MSI_OR_MSIX) 2427 return 0; 2428 #else 2429 device_t dev = adapter->dev; 2430 int rid, want, queues, msgs; 2431 2432 /* Override by tuneable */ 2433 if (ixgbe_enable_msix == 0) 2434 goto msi; 2435 2436 /* First try MSI/X */ 2437 rid = PCI_BAR(MSIX_82598_BAR); 2438 adapter->msix_mem = bus_alloc_resource_any(dev, 2439 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2440 if (!adapter->msix_mem) { 2441 rid += 4; /* 82599 maps in higher BAR */ 2442 adapter->msix_mem = bus_alloc_resource_any(dev, 2443 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2444 } 2445 if (!adapter->msix_mem) { 2446 /* May not be enabled */ 2447 device_printf(adapter->dev, 2448 "Unable to map MSIX table \n"); 2449 goto msi; 2450 } 2451 2452 msgs = pci_msix_count(dev); 2453 if (msgs == 0) { /* system has msix disabled */ 2454 bus_release_resource(dev, SYS_RES_MEMORY, 2455 rid, adapter->msix_mem); 2456 adapter->msix_mem = NULL; 2457 goto msi; 2458 } 2459 2460 /* Figure out a reasonable auto config value */ 2461 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2462 2463 if (ixgbe_num_queues != 0) 2464 queues = ixgbe_num_queues; 2465 /* Set max queues to 8 when autoconfiguring */ 2466 else if ((ixgbe_num_queues == 0) && (queues > 8)) 2467 queues = 8; 2468 2469 /* 2470 ** Want one vector (RX/TX pair) per queue 2471 ** plus an additional for Link. 2472 */ 2473 want = queues + 1; 2474 if (msgs >= want) 2475 msgs = want; 2476 else { 2477 device_printf(adapter->dev, 2478 "MSIX Configuration Problem, " 2479 "%d vectors but %d queues wanted!\n", 2480 msgs, want); 2481 return (0); /* Will go to Legacy setup */ 2482 } 2483 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2484 device_printf(adapter->dev, 2485 "Using MSIX interrupts with %d vectors\n", msgs); 2486 adapter->num_queues = queues; 2487 return (msgs); 2488 } 2489 msi: 2490 msgs = pci_msi_count(dev); 2491 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2492 device_printf(adapter->dev,"Using MSI interrupt\n"); 2493 return (msgs); 2494 #endif 2495 } 2496 2497 2498 static int 2499 ixgbe_allocate_pci_resources(struct adapter *adapter, const struct pci_attach_args *pa) 2500 { 2501 pcireg_t memtype; 2502 device_t dev = adapter->dev; 2503 bus_addr_t addr; 2504 int flags; 2505 2506 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 2507 switch (memtype) { 2508 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 2509 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 2510 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 2511 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 2512 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 2513 goto map_err; 2514 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 2515 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 2516 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 2517 } 2518 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 2519 adapter->osdep.mem_size, flags, 2520 &adapter->osdep.mem_bus_space_handle) != 0) { 2521 map_err: 2522 adapter->osdep.mem_size = 0; 2523 aprint_error_dev(dev, "unable to map BAR0\n"); 2524 return ENXIO; 2525 } 2526 break; 2527 default: 2528 aprint_error_dev(dev, "unexpected type on BAR0\n"); 2529 return ENXIO; 2530 } 2531 2532 /* Legacy defaults */ 2533 adapter->num_queues = 1; 2534 adapter->hw.back = &adapter->osdep; 2535 2536 /* 2537 ** Now setup MSI or MSI/X, should 2538 ** return us the number of supported 2539 ** vectors. (Will be 1 for MSI) 2540 */ 2541 adapter->msix = ixgbe_setup_msix(adapter); 2542 return (0); 2543 } 2544 2545 static void 2546 ixgbe_free_pci_resources(struct adapter * adapter) 2547 { 2548 #if defined(NETBSD_MSI_OR_MSIX) 2549 struct ix_queue *que = adapter->queues; 2550 #endif 2551 device_t dev = adapter->dev; 2552 int rid, memrid; 2553 2554 printf("%s: enter %s\n", device_xname(dev), __func__); 2555 2556 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 2557 memrid = PCI_BAR(MSIX_82598_BAR); 2558 else 2559 memrid = PCI_BAR(MSIX_82599_BAR); 2560 2561 #if defined(NETBSD_MSI_OR_MSIX) 2562 /* 2563 ** There is a slight possibility of a failure mode 2564 ** in attach that will result in entering this function 2565 ** before interrupt resources have been initialized, and 2566 ** in that case we do not want to execute the loops below 2567 ** We can detect this reliably by the state of the adapter 2568 ** res pointer. 2569 */ 2570 if (adapter->res == NULL) 2571 goto mem; 2572 2573 /* 2574 ** Release all msix queue resources: 2575 */ 2576 for (int i = 0; i < adapter->num_queues; i++, que++) { 2577 rid = que->msix + 1; 2578 if (que->tag != NULL) { 2579 bus_teardown_intr(dev, que->res, que->tag); 2580 que->tag = NULL; 2581 } 2582 if (que->res != NULL) 2583 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2584 } 2585 #endif 2586 2587 /* Clean the Legacy or Link interrupt last */ 2588 if (adapter->linkvec) /* we are doing MSIX */ 2589 rid = adapter->linkvec + 1; 2590 else 2591 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2592 2593 printf("%s: disestablishing interrupt handler\n", device_xname(dev)); 2594 pci_intr_disestablish(adapter->osdep.pc, adapter->osdep.intr); 2595 adapter->osdep.intr = NULL; 2596 2597 #if defined(NETBSD_MSI_OR_MSIX) 2598 mem: 2599 if (adapter->msix) 2600 pci_release_msi(dev); 2601 2602 if (adapter->msix_mem != NULL) 2603 bus_release_resource(dev, SYS_RES_MEMORY, 2604 memrid, adapter->msix_mem); 2605 #endif 2606 2607 if (adapter->osdep.mem_size != 0) { 2608 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 2609 adapter->osdep.mem_bus_space_handle, 2610 adapter->osdep.mem_size); 2611 } 2612 2613 return; 2614 } 2615 2616 /********************************************************************* 2617 * 2618 * Setup networking device structure and register an interface. 2619 * 2620 **********************************************************************/ 2621 static int 2622 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2623 { 2624 struct ethercom *ec = &adapter->osdep.ec; 2625 struct ixgbe_hw *hw = &adapter->hw; 2626 struct ifnet *ifp; 2627 2628 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2629 2630 ifp = adapter->ifp = &ec->ec_if; 2631 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 2632 ifp->if_mtu = ETHERMTU; 2633 ifp->if_baudrate = 1000000000; 2634 ifp->if_init = ixgbe_init; 2635 ifp->if_stop = ixgbe_ifstop; 2636 ifp->if_softc = adapter; 2637 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2638 ifp->if_ioctl = ixgbe_ioctl; 2639 ifp->if_start = ixgbe_start; 2640 #if __FreeBSD_version >= 800000 2641 ifp->if_transmit = ixgbe_mq_start; 2642 ifp->if_qflush = ixgbe_qflush; 2643 #endif 2644 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 2645 2646 if_attach(ifp); 2647 ether_ifattach(ifp, adapter->hw.mac.addr); 2648 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 2649 2650 adapter->max_frame_size = 2651 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2652 2653 /* 2654 * Tell the upper layer(s) we support long frames. 2655 */ 2656 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2657 2658 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 2659 ifp->if_capenable = 0; 2660 2661 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 2662 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU; 2663 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 2664 ec->ec_capenable = ec->ec_capabilities; 2665 2666 /* Don't enable LRO by default */ 2667 ifp->if_capabilities |= IFCAP_LRO; 2668 2669 /* 2670 ** Dont turn this on by default, if vlans are 2671 ** created on another pseudo device (eg. lagg) 2672 ** then vlan events are not passed thru, breaking 2673 ** operation, but with HW FILTER off it works. If 2674 ** using vlans directly on the em driver you can 2675 ** enable this and get full hardware tag filtering. 2676 */ 2677 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 2678 2679 /* 2680 * Specify the media types supported by this adapter and register 2681 * callbacks to update media and link information 2682 */ 2683 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2684 ixgbe_media_status); 2685 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL); 2686 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics); 2687 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2688 ifmedia_add(&adapter->media, 2689 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2690 ifmedia_add(&adapter->media, 2691 IFM_ETHER | IFM_1000_T, 0, NULL); 2692 } 2693 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2694 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2695 2696 return (0); 2697 } 2698 2699 static void 2700 ixgbe_config_link(struct adapter *adapter) 2701 { 2702 struct ixgbe_hw *hw = &adapter->hw; 2703 u32 autoneg, err = 0; 2704 bool sfp, negotiate; 2705 2706 sfp = ixgbe_is_sfp(hw); 2707 2708 if (sfp) { 2709 if (hw->phy.multispeed_fiber) { 2710 hw->mac.ops.setup_sfp(hw); 2711 ixgbe_enable_tx_laser(hw); 2712 softint_schedule(adapter->msf_si); 2713 } else { 2714 softint_schedule(adapter->mod_si); 2715 } 2716 } else { 2717 if (hw->mac.ops.check_link) 2718 err = ixgbe_check_link(hw, &autoneg, 2719 &adapter->link_up, FALSE); 2720 if (err) 2721 goto out; 2722 autoneg = hw->phy.autoneg_advertised; 2723 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 2724 err = hw->mac.ops.get_link_capabilities(hw, 2725 &autoneg, &negotiate); 2726 if (err) 2727 goto out; 2728 if (hw->mac.ops.setup_link) 2729 err = hw->mac.ops.setup_link(hw, autoneg, 2730 negotiate, adapter->link_up); 2731 } 2732 out: 2733 return; 2734 } 2735 2736 /******************************************************************** 2737 * Manage DMA'able memory. 2738 *******************************************************************/ 2739 2740 static int 2741 ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, 2742 struct ixgbe_dma_alloc *dma, const int mapflags) 2743 { 2744 device_t dev = adapter->dev; 2745 int r, rsegs; 2746 2747 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2748 DBA_ALIGN, 0, /* alignment, bounds */ 2749 size, /* maxsize */ 2750 1, /* nsegments */ 2751 size, /* maxsegsize */ 2752 BUS_DMA_ALLOCNOW, /* flags */ 2753 &dma->dma_tag); 2754 if (r != 0) { 2755 aprint_error_dev(dev, 2756 "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r); 2757 goto fail_0; 2758 } 2759 2760 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2761 size, 2762 dma->dma_tag->dt_alignment, 2763 dma->dma_tag->dt_boundary, 2764 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2765 if (r != 0) { 2766 aprint_error_dev(dev, 2767 "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); 2768 goto fail_1; 2769 } 2770 2771 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2772 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2773 if (r != 0) { 2774 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2775 __func__, r); 2776 goto fail_2; 2777 } 2778 2779 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2780 if (r != 0) { 2781 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2782 __func__, r); 2783 goto fail_3; 2784 } 2785 2786 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2787 size, 2788 NULL, 2789 mapflags | BUS_DMA_NOWAIT); 2790 if (r != 0) { 2791 aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", 2792 __func__, r); 2793 goto fail_4; 2794 } 2795 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2796 dma->dma_size = size; 2797 return 0; 2798 fail_4: 2799 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2800 fail_3: 2801 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2802 fail_2: 2803 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2804 fail_1: 2805 ixgbe_dma_tag_destroy(dma->dma_tag); 2806 fail_0: 2807 return r; 2808 } 2809 2810 static void 2811 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2812 { 2813 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2814 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2815 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2816 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2817 ixgbe_dma_tag_destroy(dma->dma_tag); 2818 } 2819 2820 2821 /********************************************************************* 2822 * 2823 * Allocate memory for the transmit and receive rings, and then 2824 * the descriptors associated with each, called only once at attach. 2825 * 2826 **********************************************************************/ 2827 static int 2828 ixgbe_allocate_queues(struct adapter *adapter) 2829 { 2830 device_t dev = adapter->dev; 2831 struct ix_queue *que; 2832 struct tx_ring *txr; 2833 struct rx_ring *rxr; 2834 int rsize, tsize, error = IXGBE_SUCCESS; 2835 int txconf = 0, rxconf = 0; 2836 2837 /* First allocate the top level queue structs */ 2838 if (!(adapter->queues = 2839 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2840 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2841 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2842 error = ENOMEM; 2843 goto fail; 2844 } 2845 2846 /* First allocate the TX ring struct memory */ 2847 if (!(adapter->tx_rings = 2848 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2849 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2850 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2851 error = ENOMEM; 2852 goto tx_fail; 2853 } 2854 2855 /* Next allocate the RX */ 2856 if (!(adapter->rx_rings = 2857 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2858 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2859 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2860 error = ENOMEM; 2861 goto rx_fail; 2862 } 2863 2864 /* For the ring itself */ 2865 tsize = roundup2(adapter->num_tx_desc * 2866 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2867 2868 /* 2869 * Now set up the TX queues, txconf is needed to handle the 2870 * possibility that things fail midcourse and we need to 2871 * undo memory gracefully 2872 */ 2873 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2874 /* Set up some basics */ 2875 txr = &adapter->tx_rings[i]; 2876 txr->adapter = adapter; 2877 txr->me = i; 2878 2879 /* Initialize the TX side lock */ 2880 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2881 device_xname(dev), txr->me); 2882 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2883 2884 if (ixgbe_dma_malloc(adapter, tsize, 2885 &txr->txdma, BUS_DMA_NOWAIT)) { 2886 aprint_error_dev(dev, 2887 "Unable to allocate TX Descriptor memory\n"); 2888 error = ENOMEM; 2889 goto err_tx_desc; 2890 } 2891 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2892 bzero((void *)txr->tx_base, tsize); 2893 2894 /* Now allocate transmit buffers for the ring */ 2895 if (ixgbe_allocate_transmit_buffers(txr)) { 2896 aprint_error_dev(dev, 2897 "Critical Failure setting up transmit buffers\n"); 2898 error = ENOMEM; 2899 goto err_tx_desc; 2900 } 2901 #if __FreeBSD_version >= 800000 2902 /* Allocate a buf ring */ 2903 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 2904 M_WAITOK, &txr->tx_mtx); 2905 if (txr->br == NULL) { 2906 aprint_error_dev(dev, 2907 "Critical Failure setting up buf ring\n"); 2908 error = ENOMEM; 2909 goto err_tx_desc; 2910 } 2911 #endif 2912 } 2913 2914 /* 2915 * Next the RX queues... 2916 */ 2917 rsize = roundup2(adapter->num_rx_desc * 2918 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2919 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2920 rxr = &adapter->rx_rings[i]; 2921 /* Set up some basics */ 2922 rxr->adapter = adapter; 2923 rxr->me = i; 2924 2925 /* Initialize the RX side lock */ 2926 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2927 device_xname(dev), rxr->me); 2928 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2929 2930 if (ixgbe_dma_malloc(adapter, rsize, 2931 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2932 aprint_error_dev(dev, 2933 "Unable to allocate RxDescriptor memory\n"); 2934 error = ENOMEM; 2935 goto err_rx_desc; 2936 } 2937 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2938 bzero((void *)rxr->rx_base, rsize); 2939 2940 /* Allocate receive buffers for the ring*/ 2941 if (ixgbe_allocate_receive_buffers(rxr)) { 2942 aprint_error_dev(dev, 2943 "Critical Failure setting up receive buffers\n"); 2944 error = ENOMEM; 2945 goto err_rx_desc; 2946 } 2947 } 2948 2949 /* 2950 ** Finally set up the queue holding structs 2951 */ 2952 for (int i = 0; i < adapter->num_queues; i++) { 2953 que = &adapter->queues[i]; 2954 que->adapter = adapter; 2955 que->txr = &adapter->tx_rings[i]; 2956 que->rxr = &adapter->rx_rings[i]; 2957 } 2958 2959 return (0); 2960 2961 err_rx_desc: 2962 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2963 ixgbe_dma_free(adapter, &rxr->rxdma); 2964 err_tx_desc: 2965 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2966 ixgbe_dma_free(adapter, &txr->txdma); 2967 free(adapter->rx_rings, M_DEVBUF); 2968 rx_fail: 2969 free(adapter->tx_rings, M_DEVBUF); 2970 tx_fail: 2971 free(adapter->queues, M_DEVBUF); 2972 fail: 2973 return (error); 2974 } 2975 2976 /********************************************************************* 2977 * 2978 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2979 * the information needed to transmit a packet on the wire. This is 2980 * called only once at attach, setup is done every reset. 2981 * 2982 **********************************************************************/ 2983 static int 2984 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2985 { 2986 struct adapter *adapter = txr->adapter; 2987 device_t dev = adapter->dev; 2988 struct ixgbe_tx_buf *txbuf; 2989 int error, i; 2990 2991 /* 2992 * Setup DMA descriptor areas. 2993 */ 2994 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2995 1, 0, /* alignment, bounds */ 2996 IXGBE_TSO_SIZE, /* maxsize */ 2997 adapter->num_segs, /* nsegments */ 2998 PAGE_SIZE, /* maxsegsize */ 2999 0, /* flags */ 3000 &txr->txtag))) { 3001 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 3002 goto fail; 3003 } 3004 3005 if (!(txr->tx_buffers = 3006 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 3007 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3008 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 3009 error = ENOMEM; 3010 goto fail; 3011 } 3012 3013 /* Create the descriptor buffer dma maps */ 3014 txbuf = txr->tx_buffers; 3015 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3016 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 3017 if (error != 0) { 3018 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 3019 goto fail; 3020 } 3021 } 3022 3023 return 0; 3024 fail: 3025 /* We free all, it handles case where we are in the middle */ 3026 ixgbe_free_transmit_structures(adapter); 3027 return (error); 3028 } 3029 3030 /********************************************************************* 3031 * 3032 * Initialize a transmit ring. 3033 * 3034 **********************************************************************/ 3035 static void 3036 ixgbe_setup_transmit_ring(struct tx_ring *txr) 3037 { 3038 struct adapter *adapter = txr->adapter; 3039 struct ixgbe_tx_buf *txbuf; 3040 int i; 3041 3042 /* Clear the old ring contents */ 3043 IXGBE_TX_LOCK(txr); 3044 bzero((void *)txr->tx_base, 3045 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 3046 /* Reset indices */ 3047 txr->next_avail_desc = 0; 3048 txr->next_to_clean = 0; 3049 3050 /* Free any existing tx buffers. */ 3051 txbuf = txr->tx_buffers; 3052 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3053 if (txbuf->m_head != NULL) { 3054 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 3055 0, txbuf->m_head->m_pkthdr.len, 3056 BUS_DMASYNC_POSTWRITE); 3057 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 3058 m_freem(txbuf->m_head); 3059 txbuf->m_head = NULL; 3060 } 3061 /* Clear the EOP index */ 3062 txbuf->eop_index = -1; 3063 } 3064 3065 #ifdef IXGBE_FDIR 3066 /* Set the rate at which we sample packets */ 3067 if (adapter->hw.mac.type != ixgbe_mac_82598EB) 3068 txr->atr_sample = atr_sample_rate; 3069 #endif 3070 3071 /* Set number of descriptors available */ 3072 txr->tx_avail = adapter->num_tx_desc; 3073 3074 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3075 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3076 IXGBE_TX_UNLOCK(txr); 3077 } 3078 3079 /********************************************************************* 3080 * 3081 * Initialize all transmit rings. 3082 * 3083 **********************************************************************/ 3084 static int 3085 ixgbe_setup_transmit_structures(struct adapter *adapter) 3086 { 3087 struct tx_ring *txr = adapter->tx_rings; 3088 3089 for (int i = 0; i < adapter->num_queues; i++, txr++) 3090 ixgbe_setup_transmit_ring(txr); 3091 3092 return (0); 3093 } 3094 3095 /********************************************************************* 3096 * 3097 * Enable transmit unit. 3098 * 3099 **********************************************************************/ 3100 static void 3101 ixgbe_initialize_transmit_units(struct adapter *adapter) 3102 { 3103 struct tx_ring *txr = adapter->tx_rings; 3104 struct ixgbe_hw *hw = &adapter->hw; 3105 3106 /* Setup the Base and Length of the Tx Descriptor Ring */ 3107 3108 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3109 u64 tdba = txr->txdma.dma_paddr; 3110 u32 txctrl; 3111 3112 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 3113 (tdba & 0x00000000ffffffffULL)); 3114 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 3115 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 3116 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 3117 3118 /* Setup the HW Tx Head and Tail descriptor pointers */ 3119 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 3120 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 3121 3122 /* Setup Transmit Descriptor Cmd Settings */ 3123 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 3124 txr->queue_status = IXGBE_QUEUE_IDLE; 3125 3126 /* Disable Head Writeback */ 3127 switch (hw->mac.type) { 3128 case ixgbe_mac_82598EB: 3129 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 3130 break; 3131 case ixgbe_mac_82599EB: 3132 default: 3133 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 3134 break; 3135 } 3136 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 3137 switch (hw->mac.type) { 3138 case ixgbe_mac_82598EB: 3139 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 3140 break; 3141 case ixgbe_mac_82599EB: 3142 default: 3143 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 3144 break; 3145 } 3146 3147 } 3148 3149 if (hw->mac.type != ixgbe_mac_82598EB) { 3150 u32 dmatxctl, rttdcs; 3151 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3152 dmatxctl |= IXGBE_DMATXCTL_TE; 3153 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3154 /* Disable arbiter to set MTQC */ 3155 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3156 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3157 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3158 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 3159 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3160 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3161 } 3162 3163 return; 3164 } 3165 3166 /********************************************************************* 3167 * 3168 * Free all transmit rings. 3169 * 3170 **********************************************************************/ 3171 static void 3172 ixgbe_free_transmit_structures(struct adapter *adapter) 3173 { 3174 struct tx_ring *txr = adapter->tx_rings; 3175 3176 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3177 IXGBE_TX_LOCK(txr); 3178 ixgbe_free_transmit_buffers(txr); 3179 ixgbe_dma_free(adapter, &txr->txdma); 3180 IXGBE_TX_UNLOCK(txr); 3181 IXGBE_TX_LOCK_DESTROY(txr); 3182 } 3183 free(adapter->tx_rings, M_DEVBUF); 3184 } 3185 3186 /********************************************************************* 3187 * 3188 * Free transmit ring related data structures. 3189 * 3190 **********************************************************************/ 3191 static void 3192 ixgbe_free_transmit_buffers(struct tx_ring *txr) 3193 { 3194 struct adapter *adapter = txr->adapter; 3195 struct ixgbe_tx_buf *tx_buffer; 3196 int i; 3197 3198 INIT_DEBUGOUT("free_transmit_ring: begin"); 3199 3200 if (txr->tx_buffers == NULL) 3201 return; 3202 3203 tx_buffer = txr->tx_buffers; 3204 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3205 if (tx_buffer->m_head != NULL) { 3206 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 3207 0, tx_buffer->m_head->m_pkthdr.len, 3208 BUS_DMASYNC_POSTWRITE); 3209 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3210 m_freem(tx_buffer->m_head); 3211 tx_buffer->m_head = NULL; 3212 if (tx_buffer->map != NULL) { 3213 ixgbe_dmamap_destroy(txr->txtag, 3214 tx_buffer->map); 3215 tx_buffer->map = NULL; 3216 } 3217 } else if (tx_buffer->map != NULL) { 3218 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3219 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 3220 tx_buffer->map = NULL; 3221 } 3222 } 3223 #if __FreeBSD_version >= 800000 3224 if (txr->br != NULL) 3225 buf_ring_free(txr->br, M_DEVBUF); 3226 #endif 3227 if (txr->tx_buffers != NULL) { 3228 free(txr->tx_buffers, M_DEVBUF); 3229 txr->tx_buffers = NULL; 3230 } 3231 if (txr->txtag != NULL) { 3232 ixgbe_dma_tag_destroy(txr->txtag); 3233 txr->txtag = NULL; 3234 } 3235 return; 3236 } 3237 3238 /********************************************************************* 3239 * 3240 * Advanced Context Descriptor setup for VLAN or L4 CSUM 3241 * 3242 **********************************************************************/ 3243 3244 static u32 3245 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3246 { 3247 struct m_tag *mtag; 3248 struct adapter *adapter = txr->adapter; 3249 struct ethercom *ec = &adapter->osdep.ec; 3250 struct ixgbe_adv_tx_context_desc *TXD; 3251 struct ixgbe_tx_buf *tx_buffer; 3252 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3253 struct ether_vlan_header *eh; 3254 struct ip ip; 3255 struct ip6_hdr ip6; 3256 int ehdrlen, ip_hlen = 0; 3257 u16 etype; 3258 u8 ipproto = 0; 3259 bool offload; 3260 int ctxd = txr->next_avail_desc; 3261 u16 vtag = 0; 3262 3263 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 3264 3265 tx_buffer = &txr->tx_buffers[ctxd]; 3266 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3267 3268 /* 3269 ** In advanced descriptors the vlan tag must 3270 ** be placed into the descriptor itself. 3271 */ 3272 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3273 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3274 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3275 } else if (!offload) 3276 return 0; 3277 3278 /* 3279 * Determine where frame payload starts. 3280 * Jump over vlan headers if already present, 3281 * helpful for QinQ too. 3282 */ 3283 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 3284 eh = mtod(mp, struct ether_vlan_header *); 3285 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3286 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3287 etype = ntohs(eh->evl_proto); 3288 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3289 } else { 3290 etype = ntohs(eh->evl_encap_proto); 3291 ehdrlen = ETHER_HDR_LEN; 3292 } 3293 3294 /* Set the ether header length */ 3295 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3296 3297 switch (etype) { 3298 case ETHERTYPE_IP: 3299 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 3300 ip_hlen = ip.ip_hl << 2; 3301 ipproto = ip.ip_p; 3302 #if 0 3303 ip.ip_sum = 0; 3304 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 3305 #else 3306 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3307 ip.ip_sum == 0); 3308 #endif 3309 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3310 break; 3311 case ETHERTYPE_IPV6: 3312 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 3313 ip_hlen = sizeof(ip6); 3314 ipproto = ip6.ip6_nxt; 3315 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3316 break; 3317 default: 3318 break; 3319 } 3320 3321 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 3322 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 3323 3324 vlan_macip_lens |= ip_hlen; 3325 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3326 3327 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 3328 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3329 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3330 KASSERT(ipproto == IPPROTO_TCP); 3331 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 3332 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 3333 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3334 KASSERT(ipproto == IPPROTO_UDP); 3335 } 3336 3337 /* Now copy bits into descriptor */ 3338 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3339 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3340 TXD->seqnum_seed = htole32(0); 3341 TXD->mss_l4len_idx = htole32(0); 3342 3343 tx_buffer->m_head = NULL; 3344 tx_buffer->eop_index = -1; 3345 3346 /* We've consumed the first desc, adjust counters */ 3347 if (++ctxd == adapter->num_tx_desc) 3348 ctxd = 0; 3349 txr->next_avail_desc = ctxd; 3350 --txr->tx_avail; 3351 3352 return olinfo; 3353 } 3354 3355 /********************************************************************** 3356 * 3357 * Setup work for hardware segmentation offload (TSO) on 3358 * adapters using advanced tx descriptors 3359 * 3360 **********************************************************************/ 3361 static bool 3362 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 3363 { 3364 struct m_tag *mtag; 3365 struct adapter *adapter = txr->adapter; 3366 struct ethercom *ec = &adapter->osdep.ec; 3367 struct ixgbe_adv_tx_context_desc *TXD; 3368 struct ixgbe_tx_buf *tx_buffer; 3369 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3370 u32 mss_l4len_idx = 0; 3371 u16 vtag = 0; 3372 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 3373 struct ether_vlan_header *eh; 3374 struct ip *ip; 3375 struct tcphdr *th; 3376 3377 3378 /* 3379 * Determine where frame payload starts. 3380 * Jump over vlan headers if already present 3381 */ 3382 eh = mtod(mp, struct ether_vlan_header *); 3383 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3384 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3385 else 3386 ehdrlen = ETHER_HDR_LEN; 3387 3388 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3389 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3390 return FALSE; 3391 3392 ctxd = txr->next_avail_desc; 3393 tx_buffer = &txr->tx_buffers[ctxd]; 3394 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3395 3396 ip = (struct ip *)(mp->m_data + ehdrlen); 3397 if (ip->ip_p != IPPROTO_TCP) 3398 return FALSE; /* 0 */ 3399 ip->ip_sum = 0; 3400 ip_hlen = ip->ip_hl << 2; 3401 th = (struct tcphdr *)((char *)ip + ip_hlen); 3402 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 3403 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3404 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3405 tcp_hlen = th->th_off << 2; 3406 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3407 3408 /* This is used in the transmit desc in encap */ 3409 *paylen = mp->m_pkthdr.len - hdrlen; 3410 3411 /* VLAN MACLEN IPLEN */ 3412 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3413 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3414 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3415 } 3416 3417 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3418 vlan_macip_lens |= ip_hlen; 3419 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3420 3421 /* ADV DTYPE TUCMD */ 3422 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3423 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3424 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3425 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3426 3427 3428 /* MSS L4LEN IDX */ 3429 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 3430 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 3431 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3432 3433 TXD->seqnum_seed = htole32(0); 3434 tx_buffer->m_head = NULL; 3435 tx_buffer->eop_index = -1; 3436 3437 if (++ctxd == adapter->num_tx_desc) 3438 ctxd = 0; 3439 3440 txr->tx_avail--; 3441 txr->next_avail_desc = ctxd; 3442 return TRUE; 3443 } 3444 3445 #ifdef IXGBE_FDIR 3446 /* 3447 ** This routine parses packet headers so that Flow 3448 ** Director can make a hashed filter table entry 3449 ** allowing traffic flows to be identified and kept 3450 ** on the same cpu. This would be a performance 3451 ** hit, but we only do it at IXGBE_FDIR_RATE of 3452 ** packets. 3453 */ 3454 static void 3455 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 3456 { 3457 struct adapter *adapter = txr->adapter; 3458 struct ix_queue *que; 3459 struct ip *ip; 3460 struct tcphdr *th; 3461 struct udphdr *uh; 3462 struct ether_vlan_header *eh; 3463 union ixgbe_atr_hash_dword input = {.dword = 0}; 3464 union ixgbe_atr_hash_dword common = {.dword = 0}; 3465 int ehdrlen, ip_hlen; 3466 u16 etype; 3467 3468 eh = mtod(mp, struct ether_vlan_header *); 3469 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3470 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3471 etype = eh->evl_proto; 3472 } else { 3473 ehdrlen = ETHER_HDR_LEN; 3474 etype = eh->evl_encap_proto; 3475 } 3476 3477 /* Only handling IPv4 */ 3478 if (etype != htons(ETHERTYPE_IP)) 3479 return; 3480 3481 ip = (struct ip *)(mp->m_data + ehdrlen); 3482 ip_hlen = ip->ip_hl << 2; 3483 3484 /* check if we're UDP or TCP */ 3485 switch (ip->ip_p) { 3486 case IPPROTO_TCP: 3487 th = (struct tcphdr *)((char *)ip + ip_hlen); 3488 /* src and dst are inverted */ 3489 common.port.dst ^= th->th_sport; 3490 common.port.src ^= th->th_dport; 3491 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 3492 break; 3493 case IPPROTO_UDP: 3494 uh = (struct udphdr *)((char *)ip + ip_hlen); 3495 /* src and dst are inverted */ 3496 common.port.dst ^= uh->uh_sport; 3497 common.port.src ^= uh->uh_dport; 3498 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 3499 break; 3500 default: 3501 return; 3502 } 3503 3504 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 3505 if (mp->m_pkthdr.ether_vtag) 3506 common.flex_bytes ^= htons(ETHERTYPE_VLAN); 3507 else 3508 common.flex_bytes ^= etype; 3509 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 3510 3511 que = &adapter->queues[txr->me]; 3512 /* 3513 ** This assumes the Rx queue and Tx 3514 ** queue are bound to the same CPU 3515 */ 3516 ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 3517 input, common, que->msix); 3518 } 3519 #endif /* IXGBE_FDIR */ 3520 3521 /********************************************************************** 3522 * 3523 * Examine each tx_buffer in the used queue. If the hardware is done 3524 * processing the packet then free associated resources. The 3525 * tx_buffer is put back on the free queue. 3526 * 3527 **********************************************************************/ 3528 static bool 3529 ixgbe_txeof(struct tx_ring *txr) 3530 { 3531 struct adapter *adapter = txr->adapter; 3532 struct ifnet *ifp = adapter->ifp; 3533 u32 first, last, done, processed; 3534 struct ixgbe_tx_buf *tx_buffer; 3535 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 3536 struct timeval now, elapsed; 3537 3538 KASSERT(mutex_owned(&txr->tx_mtx)); 3539 3540 if (txr->tx_avail == adapter->num_tx_desc) { 3541 txr->queue_status = IXGBE_QUEUE_IDLE; 3542 return false; 3543 } 3544 3545 processed = 0; 3546 first = txr->next_to_clean; 3547 tx_buffer = &txr->tx_buffers[first]; 3548 /* For cleanup we just use legacy struct */ 3549 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3550 last = tx_buffer->eop_index; 3551 if (last == -1) 3552 return false; 3553 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3554 3555 /* 3556 ** Get the index of the first descriptor 3557 ** BEYOND the EOP and call that 'done'. 3558 ** I do this so the comparison in the 3559 ** inner while loop below can be simple 3560 */ 3561 if (++last == adapter->num_tx_desc) last = 0; 3562 done = last; 3563 3564 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3565 BUS_DMASYNC_POSTREAD); 3566 /* 3567 ** Only the EOP descriptor of a packet now has the DD 3568 ** bit set, this is what we look for... 3569 */ 3570 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 3571 /* We clean the range of the packet */ 3572 while (first != done) { 3573 tx_desc->upper.data = 0; 3574 tx_desc->lower.data = 0; 3575 tx_desc->buffer_addr = 0; 3576 ++txr->tx_avail; 3577 ++processed; 3578 3579 if (tx_buffer->m_head) { 3580 txr->bytes += 3581 tx_buffer->m_head->m_pkthdr.len; 3582 bus_dmamap_sync(txr->txtag->dt_dmat, 3583 tx_buffer->map, 3584 0, tx_buffer->m_head->m_pkthdr.len, 3585 BUS_DMASYNC_POSTWRITE); 3586 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3587 m_freem(tx_buffer->m_head); 3588 tx_buffer->m_head = NULL; 3589 } 3590 tx_buffer->eop_index = -1; 3591 getmicrotime(&txr->watchdog_time); 3592 3593 if (++first == adapter->num_tx_desc) 3594 first = 0; 3595 3596 tx_buffer = &txr->tx_buffers[first]; 3597 tx_desc = 3598 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3599 } 3600 ++txr->packets; 3601 ++ifp->if_opackets; 3602 /* See if there is more work now */ 3603 last = tx_buffer->eop_index; 3604 if (last != -1) { 3605 eop_desc = 3606 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3607 /* Get next done point */ 3608 if (++last == adapter->num_tx_desc) last = 0; 3609 done = last; 3610 } else 3611 break; 3612 } 3613 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3614 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3615 3616 txr->next_to_clean = first; 3617 3618 /* 3619 ** Watchdog calculation, we know there's 3620 ** work outstanding or the first return 3621 ** would have been taken, so none processed 3622 ** for too long indicates a hang. 3623 */ 3624 getmicrotime(&now); 3625 timersub(&now, &txr->watchdog_time, &elapsed); 3626 if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG) 3627 txr->queue_status = IXGBE_QUEUE_HUNG; 3628 3629 /* 3630 * If we have enough room, clear IFF_OACTIVE to tell the stack that 3631 * it is OK to send packets. If there are no pending descriptors, 3632 * clear the timeout. Otherwise, if some descriptors have been freed, 3633 * restart the timeout. 3634 */ 3635 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 3636 ifp->if_flags &= ~IFF_OACTIVE; 3637 if (txr->tx_avail == adapter->num_tx_desc) { 3638 txr->queue_status = IXGBE_QUEUE_IDLE; 3639 return false; 3640 } 3641 } 3642 3643 return true; 3644 } 3645 3646 /********************************************************************* 3647 * 3648 * Refresh mbuf buffers for RX descriptor rings 3649 * - now keeps its own state so discards due to resource 3650 * exhaustion are unnecessary, if an mbuf cannot be obtained 3651 * it just returns, keeping its placeholder, thus it can simply 3652 * be recalled to try again. 3653 * 3654 **********************************************************************/ 3655 static void 3656 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 3657 { 3658 struct adapter *adapter = rxr->adapter; 3659 struct ixgbe_rx_buf *rxbuf; 3660 struct mbuf *mh, *mp; 3661 int i, j, error; 3662 bool refreshed = false; 3663 3664 i = j = rxr->next_to_refresh; 3665 /* Control the loop with one beyond */ 3666 if (++j == adapter->num_rx_desc) 3667 j = 0; 3668 3669 while (j != limit) { 3670 rxbuf = &rxr->rx_buffers[i]; 3671 if (rxr->hdr_split == FALSE) 3672 goto no_split; 3673 3674 if (rxbuf->m_head == NULL) { 3675 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3676 if (mh == NULL) 3677 goto update; 3678 } else 3679 mh = rxbuf->m_head; 3680 3681 mh->m_pkthdr.len = mh->m_len = MHLEN; 3682 mh->m_len = MHLEN; 3683 mh->m_flags |= M_PKTHDR; 3684 /* Get the memory mapping */ 3685 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3686 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 3687 if (error != 0) { 3688 printf("Refresh mbufs: hdr dmamap load" 3689 " failure - %d\n", error); 3690 m_free(mh); 3691 rxbuf->m_head = NULL; 3692 goto update; 3693 } 3694 rxbuf->m_head = mh; 3695 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD); 3696 rxr->rx_base[i].read.hdr_addr = 3697 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3698 3699 no_split: 3700 if (rxbuf->m_pack == NULL) { 3701 mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3702 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3703 if (mp == NULL) { 3704 rxr->no_jmbuf.ev_count++; 3705 goto update; 3706 } 3707 } else 3708 mp = rxbuf->m_pack; 3709 3710 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3711 /* Get the memory mapping */ 3712 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3713 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3714 if (error != 0) { 3715 printf("Refresh mbufs: payload dmamap load" 3716 " failure - %d\n", error); 3717 m_free(mp); 3718 rxbuf->m_pack = NULL; 3719 goto update; 3720 } 3721 rxbuf->m_pack = mp; 3722 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3723 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3724 rxr->rx_base[i].read.pkt_addr = 3725 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3726 3727 refreshed = true; 3728 /* Next is precalculated */ 3729 i = j; 3730 rxr->next_to_refresh = i; 3731 if (++j == adapter->num_rx_desc) 3732 j = 0; 3733 } 3734 update: 3735 if (refreshed) /* Update hardware tail index */ 3736 IXGBE_WRITE_REG(&adapter->hw, 3737 IXGBE_RDT(rxr->me), rxr->next_to_refresh); 3738 return; 3739 } 3740 3741 /********************************************************************* 3742 * 3743 * Allocate memory for rx_buffer structures. Since we use one 3744 * rx_buffer per received packet, the maximum number of rx_buffer's 3745 * that we'll need is equal to the number of receive descriptors 3746 * that we've allocated. 3747 * 3748 **********************************************************************/ 3749 static int 3750 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 3751 { 3752 struct adapter *adapter = rxr->adapter; 3753 device_t dev = adapter->dev; 3754 struct ixgbe_rx_buf *rxbuf; 3755 int i, bsize, error; 3756 3757 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 3758 if (!(rxr->rx_buffers = 3759 (struct ixgbe_rx_buf *) malloc(bsize, 3760 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3761 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 3762 error = ENOMEM; 3763 goto fail; 3764 } 3765 3766 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3767 1, 0, /* alignment, bounds */ 3768 MSIZE, /* maxsize */ 3769 1, /* nsegments */ 3770 MSIZE, /* maxsegsize */ 3771 0, /* flags */ 3772 &rxr->htag))) { 3773 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3774 goto fail; 3775 } 3776 3777 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3778 1, 0, /* alignment, bounds */ 3779 MJUM16BYTES, /* maxsize */ 3780 1, /* nsegments */ 3781 MJUM16BYTES, /* maxsegsize */ 3782 0, /* flags */ 3783 &rxr->ptag))) { 3784 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3785 goto fail; 3786 } 3787 3788 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3789 rxbuf = &rxr->rx_buffers[i]; 3790 error = ixgbe_dmamap_create(rxr->htag, 3791 BUS_DMA_NOWAIT, &rxbuf->hmap); 3792 if (error) { 3793 aprint_error_dev(dev, "Unable to create RX head map\n"); 3794 goto fail; 3795 } 3796 error = ixgbe_dmamap_create(rxr->ptag, 3797 BUS_DMA_NOWAIT, &rxbuf->pmap); 3798 if (error) { 3799 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 3800 goto fail; 3801 } 3802 } 3803 3804 return (0); 3805 3806 fail: 3807 /* Frees all, but can handle partial completion */ 3808 ixgbe_free_receive_structures(adapter); 3809 return (error); 3810 } 3811 3812 /* 3813 ** Used to detect a descriptor that has 3814 ** been merged by Hardware RSC. 3815 */ 3816 static inline u32 3817 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 3818 { 3819 return (le32toh(rx->wb.lower.lo_dword.data) & 3820 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 3821 } 3822 3823 /********************************************************************* 3824 * 3825 * Initialize Hardware RSC (LRO) feature on 82599 3826 * for an RX ring, this is toggled by the LRO capability 3827 * even though it is transparent to the stack. 3828 * 3829 **********************************************************************/ 3830 static void 3831 ixgbe_setup_hw_rsc(struct rx_ring *rxr) 3832 { 3833 struct adapter *adapter = rxr->adapter; 3834 struct ixgbe_hw *hw = &adapter->hw; 3835 u32 rscctrl, rdrxctl; 3836 3837 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3838 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3839 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3840 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 3841 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3842 3843 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 3844 rscctrl |= IXGBE_RSCCTL_RSCEN; 3845 /* 3846 ** Limit the total number of descriptors that 3847 ** can be combined, so it does not exceed 64K 3848 */ 3849 if (adapter->rx_mbuf_sz == MCLBYTES) 3850 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3851 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 3852 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 3853 else if (adapter->rx_mbuf_sz == MJUM9BYTES) 3854 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 3855 else /* Using 16K cluster */ 3856 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 3857 3858 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 3859 3860 /* Enable TCP header recognition */ 3861 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 3862 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 3863 IXGBE_PSRTYPE_TCPHDR)); 3864 3865 /* Disable RSC for ACK packets */ 3866 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3867 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3868 3869 rxr->hw_rsc = TRUE; 3870 } 3871 3872 3873 static void 3874 ixgbe_free_receive_ring(struct rx_ring *rxr) 3875 { 3876 struct adapter *adapter; 3877 struct ixgbe_rx_buf *rxbuf; 3878 int i; 3879 3880 adapter = rxr->adapter; 3881 for (i = 0; i < adapter->num_rx_desc; i++) { 3882 rxbuf = &rxr->rx_buffers[i]; 3883 if (rxbuf->m_head != NULL) { 3884 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 3885 BUS_DMASYNC_POSTREAD); 3886 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 3887 rxbuf->m_head->m_flags |= M_PKTHDR; 3888 m_freem(rxbuf->m_head); 3889 } 3890 if (rxbuf->m_pack != NULL) { 3891 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3892 0, rxbuf->m_pack->m_pkthdr.len, 3893 BUS_DMASYNC_POSTREAD); 3894 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 3895 rxbuf->m_pack->m_flags |= M_PKTHDR; 3896 m_freem(rxbuf->m_pack); 3897 } 3898 rxbuf->m_head = NULL; 3899 rxbuf->m_pack = NULL; 3900 } 3901 } 3902 3903 3904 /********************************************************************* 3905 * 3906 * Initialize a receive ring and its buffers. 3907 * 3908 **********************************************************************/ 3909 static int 3910 ixgbe_setup_receive_ring(struct rx_ring *rxr) 3911 { 3912 struct adapter *adapter; 3913 struct ifnet *ifp; 3914 device_t dev; 3915 struct ixgbe_rx_buf *rxbuf; 3916 #ifdef LRO 3917 struct lro_ctrl *lro = &rxr->lro; 3918 #endif /* LRO */ 3919 int rsize, error = 0; 3920 3921 adapter = rxr->adapter; 3922 ifp = adapter->ifp; 3923 dev = adapter->dev; 3924 3925 /* Clear the ring contents */ 3926 IXGBE_RX_LOCK(rxr); 3927 rsize = roundup2(adapter->num_rx_desc * 3928 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3929 bzero((void *)rxr->rx_base, rsize); 3930 3931 /* Free current RX buffer structs and their mbufs */ 3932 ixgbe_free_receive_ring(rxr); 3933 3934 /* Now reinitialize our supply of jumbo mbufs. The number 3935 * or size of jumbo mbufs may have changed. 3936 */ 3937 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 3938 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 3939 3940 /* Configure header split? */ 3941 if (ixgbe_header_split) 3942 rxr->hdr_split = TRUE; 3943 3944 /* Now replenish the mbufs */ 3945 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3946 struct mbuf *mh, *mp; 3947 3948 rxbuf = &rxr->rx_buffers[j]; 3949 /* 3950 ** Don't allocate mbufs if not 3951 ** doing header split, its wasteful 3952 */ 3953 if (rxr->hdr_split == FALSE) 3954 goto skip_head; 3955 3956 /* First the header */ 3957 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3958 if (rxbuf->m_head == NULL) { 3959 error = ENOBUFS; 3960 goto fail; 3961 } 3962 m_adj(rxbuf->m_head, ETHER_ALIGN); 3963 mh = rxbuf->m_head; 3964 mh->m_len = mh->m_pkthdr.len = MHLEN; 3965 mh->m_flags |= M_PKTHDR; 3966 /* Get the memory mapping */ 3967 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3968 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 3969 if (error != 0) /* Nothing elegant to do here */ 3970 goto fail; 3971 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 3972 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3973 /* Update descriptor */ 3974 rxr->rx_base[j].read.hdr_addr = 3975 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3976 3977 skip_head: 3978 /* Now the payload cluster */ 3979 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3980 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3981 if (rxbuf->m_pack == NULL) { 3982 error = ENOBUFS; 3983 goto fail; 3984 } 3985 mp = rxbuf->m_pack; 3986 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3987 /* Get the memory mapping */ 3988 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3989 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3990 if (error != 0) 3991 goto fail; 3992 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3993 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 3994 /* Update descriptor */ 3995 rxr->rx_base[j].read.pkt_addr = 3996 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3997 } 3998 3999 4000 /* Setup our descriptor indices */ 4001 rxr->next_to_check = 0; 4002 rxr->next_to_refresh = 0; 4003 rxr->lro_enabled = FALSE; 4004 rxr->rx_split_packets.ev_count = 0; 4005 rxr->rx_bytes.ev_count = 0; 4006 rxr->discard = FALSE; 4007 4008 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4009 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4010 4011 /* 4012 ** Now set up the LRO interface: 4013 ** 82598 uses software LRO, the 4014 ** 82599 uses a hardware assist. 4015 */ 4016 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && 4017 (ifp->if_capenable & IFCAP_RXCSUM) && 4018 (ifp->if_capenable & IFCAP_LRO)) 4019 ixgbe_setup_hw_rsc(rxr); 4020 #ifdef LRO 4021 else if (ifp->if_capenable & IFCAP_LRO) { 4022 int err = tcp_lro_init(lro); 4023 if (err) { 4024 device_printf(dev, "LRO Initialization failed!\n"); 4025 goto fail; 4026 } 4027 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 4028 rxr->lro_enabled = TRUE; 4029 lro->ifp = adapter->ifp; 4030 } 4031 #endif /* LRO */ 4032 4033 IXGBE_RX_UNLOCK(rxr); 4034 return (0); 4035 4036 fail: 4037 ixgbe_free_receive_ring(rxr); 4038 IXGBE_RX_UNLOCK(rxr); 4039 return (error); 4040 } 4041 4042 /********************************************************************* 4043 * 4044 * Initialize all receive rings. 4045 * 4046 **********************************************************************/ 4047 static int 4048 ixgbe_setup_receive_structures(struct adapter *adapter) 4049 { 4050 struct rx_ring *rxr = adapter->rx_rings; 4051 int j; 4052 4053 for (j = 0; j < adapter->num_queues; j++, rxr++) 4054 if (ixgbe_setup_receive_ring(rxr)) 4055 goto fail; 4056 4057 return (0); 4058 fail: 4059 /* 4060 * Free RX buffers allocated so far, we will only handle 4061 * the rings that completed, the failing case will have 4062 * cleaned up for itself. 'j' failed, so its the terminus. 4063 */ 4064 for (int i = 0; i < j; ++i) { 4065 rxr = &adapter->rx_rings[i]; 4066 ixgbe_free_receive_ring(rxr); 4067 } 4068 4069 return (ENOBUFS); 4070 } 4071 4072 /********************************************************************* 4073 * 4074 * Setup receive registers and features. 4075 * 4076 **********************************************************************/ 4077 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 4078 4079 static void 4080 ixgbe_initialize_receive_units(struct adapter *adapter) 4081 { 4082 int i; 4083 struct rx_ring *rxr = adapter->rx_rings; 4084 struct ixgbe_hw *hw = &adapter->hw; 4085 struct ifnet *ifp = adapter->ifp; 4086 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; 4087 u32 reta, mrqc = 0, hlreg, r[10]; 4088 4089 4090 /* 4091 * Make sure receives are disabled while 4092 * setting up the descriptor ring 4093 */ 4094 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4095 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, 4096 rxctrl & ~IXGBE_RXCTRL_RXEN); 4097 4098 /* Enable broadcasts */ 4099 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4100 fctrl |= IXGBE_FCTRL_BAM; 4101 fctrl |= IXGBE_FCTRL_DPF; 4102 fctrl |= IXGBE_FCTRL_PMCF; 4103 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4104 4105 /* Set for Jumbo Frames? */ 4106 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4107 if (ifp->if_mtu > ETHERMTU) 4108 hlreg |= IXGBE_HLREG0_JUMBOEN; 4109 else 4110 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 4111 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 4112 4113 bufsz = adapter->rx_mbuf_sz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 4114 4115 for (i = 0; i < adapter->num_queues; i++, rxr++) { 4116 u64 rdba = rxr->rxdma.dma_paddr; 4117 4118 printf("%s: queue %d rdba %" PRIx64 "\n", __func__, i, rdba); 4119 4120 /* Setup the Base and Length of the Rx Descriptor Ring */ 4121 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 4122 (rdba & 0x00000000ffffffffULL)); 4123 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 4124 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 4125 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 4126 4127 /* Set up the SRRCTL register */ 4128 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 4129 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4130 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 4131 srrctl |= bufsz; 4132 if (rxr->hdr_split) { 4133 /* Use a standard mbuf for the header */ 4134 srrctl |= ((IXGBE_RX_HDR << 4135 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 4136 & IXGBE_SRRCTL_BSIZEHDR_MASK); 4137 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4138 } else 4139 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 4140 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 4141 4142 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 4143 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 4144 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 4145 } 4146 4147 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 4148 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4149 IXGBE_PSRTYPE_UDPHDR | 4150 IXGBE_PSRTYPE_IPV4HDR | 4151 IXGBE_PSRTYPE_IPV6HDR; 4152 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 4153 } 4154 4155 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 4156 4157 /* Setup RSS */ 4158 if (adapter->num_queues > 1) { 4159 int j; 4160 reta = 0; 4161 4162 /* set up random bits */ 4163 cprng_fast(&r, sizeof(r)); 4164 4165 /* Set up the redirection table */ 4166 for (i = 0, j = 0; i < 128; i++, j++) { 4167 if (j == adapter->num_queues) j = 0; 4168 reta = (reta << 8) | (j * 0x11); 4169 if ((i & 3) == 3) 4170 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 4171 } 4172 4173 /* Now fill our hash function seeds */ 4174 for (i = 0; i < 10; i++) 4175 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), r[i]); 4176 4177 /* Perform hash on these packet types */ 4178 mrqc = IXGBE_MRQC_RSSEN 4179 | IXGBE_MRQC_RSS_FIELD_IPV4 4180 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 4181 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 4182 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 4183 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 4184 | IXGBE_MRQC_RSS_FIELD_IPV6 4185 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 4186 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 4187 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 4188 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4189 4190 /* RSS and RX IPP Checksum are mutually exclusive */ 4191 rxcsum |= IXGBE_RXCSUM_PCSD; 4192 } 4193 4194 if (ifp->if_capenable & IFCAP_RXCSUM) 4195 rxcsum |= IXGBE_RXCSUM_PCSD; 4196 4197 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 4198 rxcsum |= IXGBE_RXCSUM_IPPCSE; 4199 4200 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 4201 4202 return; 4203 } 4204 4205 /********************************************************************* 4206 * 4207 * Free all receive rings. 4208 * 4209 **********************************************************************/ 4210 static void 4211 ixgbe_free_receive_structures(struct adapter *adapter) 4212 { 4213 struct rx_ring *rxr = adapter->rx_rings; 4214 4215 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4216 #ifdef LRO 4217 struct lro_ctrl *lro = &rxr->lro; 4218 #endif /* LRO */ 4219 ixgbe_free_receive_buffers(rxr); 4220 #ifdef LRO 4221 /* Free LRO memory */ 4222 tcp_lro_free(lro); 4223 #endif /* LRO */ 4224 /* Free the ring memory as well */ 4225 ixgbe_dma_free(adapter, &rxr->rxdma); 4226 } 4227 4228 free(adapter->rx_rings, M_DEVBUF); 4229 } 4230 4231 4232 /********************************************************************* 4233 * 4234 * Free receive ring data structures 4235 * 4236 **********************************************************************/ 4237 static void 4238 ixgbe_free_receive_buffers(struct rx_ring *rxr) 4239 { 4240 struct adapter *adapter = rxr->adapter; 4241 struct ixgbe_rx_buf *rxbuf; 4242 4243 INIT_DEBUGOUT("free_receive_structures: begin"); 4244 4245 /* Cleanup any existing buffers */ 4246 if (rxr->rx_buffers != NULL) { 4247 for (int i = 0; i < adapter->num_rx_desc; i++) { 4248 rxbuf = &rxr->rx_buffers[i]; 4249 if (rxbuf->m_head != NULL) { 4250 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4251 BUS_DMASYNC_POSTREAD); 4252 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4253 rxbuf->m_head->m_flags |= M_PKTHDR; 4254 m_freem(rxbuf->m_head); 4255 } 4256 if (rxbuf->m_pack != NULL) { 4257 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4258 0, rxbuf->m_pack->m_pkthdr.len, 4259 BUS_DMASYNC_POSTREAD); 4260 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4261 rxbuf->m_pack->m_flags |= M_PKTHDR; 4262 m_freem(rxbuf->m_pack); 4263 } 4264 rxbuf->m_head = NULL; 4265 rxbuf->m_pack = NULL; 4266 if (rxbuf->hmap != NULL) { 4267 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 4268 rxbuf->hmap = NULL; 4269 } 4270 if (rxbuf->pmap != NULL) { 4271 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4272 rxbuf->pmap = NULL; 4273 } 4274 } 4275 if (rxr->rx_buffers != NULL) { 4276 free(rxr->rx_buffers, M_DEVBUF); 4277 rxr->rx_buffers = NULL; 4278 } 4279 } 4280 4281 if (rxr->htag != NULL) { 4282 ixgbe_dma_tag_destroy(rxr->htag); 4283 rxr->htag = NULL; 4284 } 4285 if (rxr->ptag != NULL) { 4286 ixgbe_dma_tag_destroy(rxr->ptag); 4287 rxr->ptag = NULL; 4288 } 4289 4290 return; 4291 } 4292 4293 static __inline void 4294 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4295 { 4296 struct ethercom *ec; 4297 struct adapter *adapter = ifp->if_softc; 4298 int s; 4299 4300 ec = &adapter->osdep.ec; 4301 4302 #ifdef LRO 4303 /* 4304 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4305 * should be computed by hardware. Also it should not have VLAN tag in 4306 * ethernet header. 4307 */ 4308 if (rxr->lro_enabled && 4309 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 4310 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4311 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4312 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 4313 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4314 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4315 /* 4316 * Send to the stack if: 4317 ** - LRO not enabled, or 4318 ** - no LRO resources, or 4319 ** - lro enqueue fails 4320 */ 4321 if (rxr->lro.lro_cnt != 0) 4322 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4323 return; 4324 } 4325 #endif /* LRO */ 4326 4327 IXGBE_RX_UNLOCK(rxr); 4328 4329 s = splnet(); 4330 /* Pass this up to any BPF listeners. */ 4331 bpf_mtap(ifp, m); 4332 (*ifp->if_input)(ifp, m); 4333 splx(s); 4334 4335 IXGBE_RX_LOCK(rxr); 4336 } 4337 4338 static __inline void 4339 ixgbe_rx_discard(struct rx_ring *rxr, int i) 4340 { 4341 struct ixgbe_rx_buf *rbuf; 4342 4343 rbuf = &rxr->rx_buffers[i]; 4344 4345 printf("%s: enter\n", __func__); 4346 4347 if (rbuf->fmp != NULL) {/* Partial chain ? */ 4348 rbuf->fmp->m_flags |= M_PKTHDR; 4349 m_freem(rbuf->fmp); 4350 rbuf->fmp = NULL; 4351 } 4352 4353 /* 4354 ** With advanced descriptors the writeback 4355 ** clobbers the buffer addrs, so its easier 4356 ** to just free the existing mbufs and take 4357 ** the normal refresh path to get new buffers 4358 ** and mapping. 4359 */ 4360 if (rbuf->m_head) { 4361 m_free(rbuf->m_head); 4362 rbuf->m_head = NULL; 4363 } 4364 4365 if (rbuf->m_pack) { 4366 m_free(rbuf->m_pack); 4367 rbuf->m_pack = NULL; 4368 } 4369 4370 return; 4371 } 4372 4373 4374 /********************************************************************* 4375 * 4376 * This routine executes in interrupt context. It replenishes 4377 * the mbufs in the descriptor and sends data which has been 4378 * dma'ed into host memory to upper layer. 4379 * 4380 * We loop at most count times if count is > 0, or until done if 4381 * count < 0. 4382 * 4383 * Return TRUE for more work, FALSE for all clean. 4384 *********************************************************************/ 4385 static bool 4386 ixgbe_rxeof(struct ix_queue *que, int count) 4387 { 4388 struct adapter *adapter = que->adapter; 4389 struct rx_ring *rxr = que->rxr; 4390 struct ifnet *ifp = adapter->ifp; 4391 #ifdef LRO 4392 struct lro_ctrl *lro = &rxr->lro; 4393 struct lro_entry *queued; 4394 #endif /* LRO */ 4395 int i, nextp, processed = 0; 4396 u32 staterr = 0; 4397 union ixgbe_adv_rx_desc *cur; 4398 struct ixgbe_rx_buf *rbuf, *nbuf; 4399 4400 IXGBE_RX_LOCK(rxr); 4401 4402 for (i = rxr->next_to_check; count != 0;) { 4403 struct mbuf *sendmp, *mh, *mp; 4404 u32 rsc, ptype; 4405 u16 hlen, plen, hdr, vtag; 4406 bool eop; 4407 4408 /* Sync the ring. */ 4409 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4410 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4411 4412 cur = &rxr->rx_base[i]; 4413 staterr = le32toh(cur->wb.upper.status_error); 4414 4415 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 4416 break; 4417 if ((ifp->if_flags & IFF_RUNNING) == 0) 4418 break; 4419 4420 count--; 4421 sendmp = NULL; 4422 nbuf = NULL; 4423 rsc = 0; 4424 cur->wb.upper.status_error = 0; 4425 rbuf = &rxr->rx_buffers[i]; 4426 mh = rbuf->m_head; 4427 mp = rbuf->m_pack; 4428 4429 plen = le16toh(cur->wb.upper.length); 4430 ptype = le32toh(cur->wb.lower.lo_dword.data) & 4431 IXGBE_RXDADV_PKTTYPE_MASK; 4432 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4433 vtag = le16toh(cur->wb.upper.vlan); 4434 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 4435 4436 /* Make sure bad packets are discarded */ 4437 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 4438 (rxr->discard)) { 4439 ifp->if_ierrors++; 4440 rxr->rx_discarded.ev_count++; 4441 if (eop) 4442 rxr->discard = FALSE; 4443 else 4444 rxr->discard = TRUE; 4445 ixgbe_rx_discard(rxr, i); 4446 goto next_desc; 4447 } 4448 4449 /* 4450 ** On 82599 which supports a hardware 4451 ** LRO (called HW RSC), packets need 4452 ** not be fragmented across sequential 4453 ** descriptors, rather the next descriptor 4454 ** is indicated in bits of the descriptor. 4455 ** This also means that we might proceses 4456 ** more than one packet at a time, something 4457 ** that has never been true before, it 4458 ** required eliminating global chain pointers 4459 ** in favor of what we are doing here. -jfv 4460 */ 4461 if (!eop) { 4462 /* 4463 ** Figure out the next descriptor 4464 ** of this frame. 4465 */ 4466 if (rxr->hw_rsc == TRUE) { 4467 rsc = ixgbe_rsc_count(cur); 4468 rxr->rsc_num += (rsc - 1); 4469 } 4470 if (rsc) { /* Get hardware index */ 4471 nextp = ((staterr & 4472 IXGBE_RXDADV_NEXTP_MASK) >> 4473 IXGBE_RXDADV_NEXTP_SHIFT); 4474 } else { /* Just sequential */ 4475 nextp = i + 1; 4476 if (nextp == adapter->num_rx_desc) 4477 nextp = 0; 4478 } 4479 nbuf = &rxr->rx_buffers[nextp]; 4480 prefetch(nbuf); 4481 } 4482 /* 4483 ** The header mbuf is ONLY used when header 4484 ** split is enabled, otherwise we get normal 4485 ** behavior, ie, both header and payload 4486 ** are DMA'd into the payload buffer. 4487 ** 4488 ** Rather than using the fmp/lmp global pointers 4489 ** we now keep the head of a packet chain in the 4490 ** buffer struct and pass this along from one 4491 ** descriptor to the next, until we get EOP. 4492 */ 4493 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 4494 /* This must be an initial descriptor */ 4495 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 4496 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 4497 if (hlen > IXGBE_RX_HDR) 4498 hlen = IXGBE_RX_HDR; 4499 mh->m_len = hlen; 4500 mh->m_flags |= M_PKTHDR; 4501 mh->m_next = NULL; 4502 mh->m_pkthdr.len = mh->m_len; 4503 /* Null buf pointer so it is refreshed */ 4504 rbuf->m_head = NULL; 4505 /* 4506 ** Check the payload length, this 4507 ** could be zero if its a small 4508 ** packet. 4509 */ 4510 if (plen > 0) { 4511 mp->m_len = plen; 4512 mp->m_next = NULL; 4513 mp->m_flags &= ~M_PKTHDR; 4514 mh->m_next = mp; 4515 mh->m_pkthdr.len += mp->m_len; 4516 /* Null buf pointer so it is refreshed */ 4517 rbuf->m_pack = NULL; 4518 rxr->rx_split_packets.ev_count++; 4519 } 4520 /* 4521 ** Now create the forward 4522 ** chain so when complete 4523 ** we wont have to. 4524 */ 4525 if (eop == 0) { 4526 /* stash the chain head */ 4527 nbuf->fmp = mh; 4528 /* Make forward chain */ 4529 if (plen) 4530 mp->m_next = nbuf->m_pack; 4531 else 4532 mh->m_next = nbuf->m_pack; 4533 } else { 4534 /* Singlet, prepare to send */ 4535 sendmp = mh; 4536 if (VLAN_ATTACHED(&adapter->osdep.ec) && 4537 (staterr & IXGBE_RXD_STAT_VP)) { 4538 /* XXX Do something reasonable on 4539 * error. 4540 */ 4541 #if 0 4542 printf("%s.%d: VLAN_INPUT_TAG\n", 4543 __func__, __LINE__); 4544 Debugger(); 4545 #endif 4546 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4547 printf("%s: could not apply VLAN " 4548 "tag", __func__)); 4549 } 4550 } 4551 } else { 4552 /* 4553 ** Either no header split, or a 4554 ** secondary piece of a fragmented 4555 ** split packet. 4556 */ 4557 mp->m_len = plen; 4558 /* 4559 ** See if there is a stored head 4560 ** that determines what we are 4561 */ 4562 sendmp = rbuf->fmp; 4563 rbuf->m_pack = rbuf->fmp = NULL; 4564 4565 if (sendmp != NULL) /* secondary frag */ 4566 sendmp->m_pkthdr.len += mp->m_len; 4567 else { 4568 /* first desc of a non-ps chain */ 4569 sendmp = mp; 4570 sendmp->m_flags |= M_PKTHDR; 4571 sendmp->m_pkthdr.len = mp->m_len; 4572 if (staterr & IXGBE_RXD_STAT_VP) { 4573 /* XXX Do something reasonable on 4574 * error. 4575 */ 4576 #if 0 4577 printf("%s.%d: VLAN_INPUT_TAG\n", 4578 __func__, __LINE__); 4579 Debugger(); 4580 #endif 4581 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4582 printf("%s: could not apply VLAN " 4583 "tag", __func__)); 4584 } 4585 } 4586 /* Pass the head pointer on */ 4587 if (eop == 0) { 4588 nbuf->fmp = sendmp; 4589 sendmp = NULL; 4590 mp->m_next = nbuf->m_pack; 4591 } 4592 } 4593 ++processed; 4594 /* Sending this frame? */ 4595 if (eop) { 4596 sendmp->m_pkthdr.rcvif = ifp; 4597 ifp->if_ipackets++; 4598 rxr->rx_packets.ev_count++; 4599 /* capture data for AIM */ 4600 rxr->bytes += sendmp->m_pkthdr.len; 4601 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 4602 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 4603 ixgbe_rx_checksum(staterr, sendmp, ptype, 4604 &adapter->stats); 4605 } 4606 #if __FreeBSD_version >= 800000 4607 sendmp->m_pkthdr.flowid = que->msix; 4608 sendmp->m_flags |= M_FLOWID; 4609 #endif 4610 } 4611 next_desc: 4612 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4613 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4614 4615 /* Advance our pointers to the next descriptor. */ 4616 if (++i == adapter->num_rx_desc) 4617 i = 0; 4618 4619 /* Now send to the stack or do LRO */ 4620 if (sendmp != NULL) { 4621 rxr->next_to_check = i; 4622 ixgbe_rx_input(rxr, ifp, sendmp, ptype); 4623 i = rxr->next_to_check; 4624 } 4625 4626 /* Every 8 descriptors we go to refresh mbufs */ 4627 if (processed == 8) { 4628 ixgbe_refresh_mbufs(rxr, i); 4629 processed = 0; 4630 } 4631 } 4632 4633 /* Refresh any remaining buf structs */ 4634 if (ixgbe_rx_unrefreshed(rxr)) 4635 ixgbe_refresh_mbufs(rxr, i); 4636 4637 rxr->next_to_check = i; 4638 4639 #ifdef LRO 4640 /* 4641 * Flush any outstanding LRO work 4642 */ 4643 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4644 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4645 tcp_lro_flush(lro, queued); 4646 } 4647 #endif /* LRO */ 4648 4649 IXGBE_RX_UNLOCK(rxr); 4650 4651 /* 4652 ** We still have cleaning to do? 4653 ** Schedule another interrupt if so. 4654 */ 4655 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 4656 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix)); 4657 return true; 4658 } 4659 4660 return false; 4661 } 4662 4663 4664 /********************************************************************* 4665 * 4666 * Verify that the hardware indicated that the checksum is valid. 4667 * Inform the stack about the status of checksum so that stack 4668 * doesn't spend time verifying the checksum. 4669 * 4670 *********************************************************************/ 4671 static void 4672 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 4673 struct ixgbe_hw_stats *stats) 4674 { 4675 u16 status = (u16) staterr; 4676 u8 errors = (u8) (staterr >> 24); 4677 bool sctp = FALSE; 4678 4679 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4680 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 4681 sctp = TRUE; 4682 4683 if (status & IXGBE_RXD_STAT_IPCS) { 4684 stats->ipcs.ev_count++; 4685 if (!(errors & IXGBE_RXD_ERR_IPE)) { 4686 /* IP Checksum Good */ 4687 mp->m_pkthdr.csum_flags = M_CSUM_IPv4; 4688 4689 } else { 4690 stats->ipcs_bad.ev_count++; 4691 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 4692 } 4693 } 4694 if (status & IXGBE_RXD_STAT_L4CS) { 4695 stats->l4cs.ev_count++; 4696 u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 4697 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 4698 mp->m_pkthdr.csum_flags |= type; 4699 } else { 4700 stats->l4cs_bad.ev_count++; 4701 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 4702 } 4703 } 4704 return; 4705 } 4706 4707 4708 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 4709 /* 4710 ** This routine is run via an vlan config EVENT, 4711 ** it enables us to use the HW Filter table since 4712 ** we can get the vlan id. This just creates the 4713 ** entry in the soft version of the VFTA, init will 4714 ** repopulate the real table. 4715 */ 4716 static void 4717 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4718 { 4719 struct adapter *adapter = ifp->if_softc; 4720 u16 index, bit; 4721 4722 if (ifp->if_softc != arg) /* Not our event */ 4723 return; 4724 4725 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4726 return; 4727 4728 IXGBE_CORE_LOCK(adapter); 4729 index = (vtag >> 5) & 0x7F; 4730 bit = vtag & 0x1F; 4731 adapter->shadow_vfta[index] |= (1 << bit); 4732 ixgbe_init_locked(adapter); 4733 IXGBE_CORE_UNLOCK(adapter); 4734 } 4735 4736 /* 4737 ** This routine is run via an vlan 4738 ** unconfig EVENT, remove our entry 4739 ** in the soft vfta. 4740 */ 4741 static void 4742 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4743 { 4744 struct adapter *adapter = ifp->if_softc; 4745 u16 index, bit; 4746 4747 if (ifp->if_softc != arg) 4748 return; 4749 4750 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4751 return; 4752 4753 IXGBE_CORE_LOCK(adapter); 4754 index = (vtag >> 5) & 0x7F; 4755 bit = vtag & 0x1F; 4756 adapter->shadow_vfta[index] &= ~(1 << bit); 4757 /* Re-init to load the changes */ 4758 ixgbe_init_locked(adapter); 4759 IXGBE_CORE_UNLOCK(adapter); 4760 } 4761 #endif 4762 4763 static void 4764 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 4765 { 4766 struct ethercom *ec = &adapter->osdep.ec; 4767 struct ixgbe_hw *hw = &adapter->hw; 4768 u32 ctrl; 4769 4770 printf("%s: %s enter\n", device_xname(adapter->dev), __func__); 4771 4772 /* 4773 ** We get here thru init_locked, meaning 4774 ** a soft reset, this has already cleared 4775 ** the VFTA and other state, so if there 4776 ** have been no vlan's registered do nothing. 4777 */ 4778 if (!VLAN_ATTACHED(&adapter->osdep.ec)) { 4779 printf("%s: no VLANs attached\n", device_xname(adapter->dev)); 4780 return; 4781 } 4782 4783 /* 4784 ** A soft reset zero's out the VFTA, so 4785 ** we need to repopulate it now. 4786 */ 4787 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 4788 if (adapter->shadow_vfta[i] != 0) 4789 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 4790 adapter->shadow_vfta[i]); 4791 4792 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4793 /* Enable the Filter Table if enabled */ 4794 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) { 4795 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 4796 ctrl |= IXGBE_VLNCTRL_VFE; 4797 printf("%s: enabled h/w VLAN filter\n", 4798 device_xname(adapter->dev)); 4799 } 4800 if (hw->mac.type == ixgbe_mac_82598EB) 4801 ctrl |= IXGBE_VLNCTRL_VME; 4802 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 4803 4804 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 4805 if (hw->mac.type != ixgbe_mac_82598EB) 4806 for (int i = 0; i < adapter->num_queues; i++) { 4807 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 4808 ctrl |= IXGBE_RXDCTL_VME; 4809 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 4810 printf("%s: enabled VLAN queue %d\n", 4811 device_xname(adapter->dev), i); 4812 } 4813 } 4814 4815 static void 4816 ixgbe_enable_intr(struct adapter *adapter) 4817 { 4818 struct ixgbe_hw *hw = &adapter->hw; 4819 struct ix_queue *que = adapter->queues; 4820 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 4821 4822 4823 /* Enable Fan Failure detection */ 4824 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4825 mask |= IXGBE_EIMS_GPI_SDP1; 4826 else { 4827 mask |= IXGBE_EIMS_ECC; 4828 mask |= IXGBE_EIMS_GPI_SDP1; 4829 mask |= IXGBE_EIMS_GPI_SDP2; 4830 #ifdef IXGBE_FDIR 4831 mask |= IXGBE_EIMS_FLOW_DIR; 4832 #endif 4833 } 4834 4835 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 4836 4837 /* With RSS we use auto clear */ 4838 if (adapter->msix_mem) { 4839 mask = IXGBE_EIMS_ENABLE_MASK; 4840 /* Don't autoclear Link */ 4841 mask &= ~IXGBE_EIMS_OTHER; 4842 mask &= ~IXGBE_EIMS_LSC; 4843 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 4844 } 4845 4846 /* 4847 ** Now enable all queues, this is done separately to 4848 ** allow for handling the extended (beyond 32) MSIX 4849 ** vectors that can be used by 82599 4850 */ 4851 for (int i = 0; i < adapter->num_queues; i++, que++) 4852 ixgbe_enable_queue(adapter, que->msix); 4853 4854 IXGBE_WRITE_FLUSH(hw); 4855 4856 return; 4857 } 4858 4859 static void 4860 ixgbe_disable_intr(struct adapter *adapter) 4861 { 4862 if (adapter->msix_mem) 4863 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 4864 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 4865 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 4866 } else { 4867 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 4868 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 4869 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 4870 } 4871 IXGBE_WRITE_FLUSH(&adapter->hw); 4872 return; 4873 } 4874 4875 u16 4876 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 4877 { 4878 switch (reg % 4) { 4879 case 0: 4880 return pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4881 __BITS(15, 0); 4882 case 2: 4883 return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag, 4884 reg - 2), __BITS(31, 16)); 4885 default: 4886 panic("%s: invalid register (%" PRIx32, __func__, reg); 4887 break; 4888 } 4889 } 4890 4891 void 4892 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 4893 { 4894 pcireg_t old; 4895 4896 switch (reg % 4) { 4897 case 0: 4898 old = pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4899 __BITS(31, 16); 4900 pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old); 4901 break; 4902 case 2: 4903 old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) & 4904 __BITS(15, 0); 4905 pci_conf_write(hw->back->pc, hw->back->tag, reg - 2, 4906 __SHIFTIN(value, __BITS(31, 16)) | old); 4907 break; 4908 default: 4909 panic("%s: invalid register (%" PRIx32, __func__, reg); 4910 break; 4911 } 4912 4913 return; 4914 } 4915 4916 /* 4917 ** Setup the correct IVAR register for a particular MSIX interrupt 4918 ** (yes this is all very magic and confusing :) 4919 ** - entry is the register array entry 4920 ** - vector is the MSIX vector for this queue 4921 ** - type is RX/TX/MISC 4922 */ 4923 static void 4924 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4925 { 4926 struct ixgbe_hw *hw = &adapter->hw; 4927 u32 ivar, index; 4928 4929 vector |= IXGBE_IVAR_ALLOC_VAL; 4930 4931 switch (hw->mac.type) { 4932 4933 case ixgbe_mac_82598EB: 4934 if (type == -1) 4935 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4936 else 4937 entry += (type * 64); 4938 index = (entry >> 2) & 0x1F; 4939 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4940 ivar &= ~(0xFF << (8 * (entry & 0x3))); 4941 ivar |= (vector << (8 * (entry & 0x3))); 4942 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4943 break; 4944 4945 case ixgbe_mac_82599EB: 4946 if (type == -1) { /* MISC IVAR */ 4947 index = (entry & 1) * 8; 4948 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4949 ivar &= ~(0xFF << index); 4950 ivar |= (vector << index); 4951 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4952 } else { /* RX/TX IVARS */ 4953 index = (16 * (entry & 1)) + (8 * type); 4954 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4955 ivar &= ~(0xFF << index); 4956 ivar |= (vector << index); 4957 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4958 } 4959 4960 default: 4961 break; 4962 } 4963 } 4964 4965 static void 4966 ixgbe_configure_ivars(struct adapter *adapter) 4967 { 4968 struct ix_queue *que = adapter->queues; 4969 u32 newitr; 4970 4971 if (ixgbe_max_interrupt_rate > 0) 4972 newitr = (8000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4973 else 4974 newitr = 0; 4975 4976 for (int i = 0; i < adapter->num_queues; i++, que++) { 4977 /* First the RX queue entry */ 4978 ixgbe_set_ivar(adapter, i, que->msix, 0); 4979 /* ... and the TX */ 4980 ixgbe_set_ivar(adapter, i, que->msix, 1); 4981 /* Set an Initial EITR value */ 4982 IXGBE_WRITE_REG(&adapter->hw, 4983 IXGBE_EITR(que->msix), newitr); 4984 } 4985 4986 /* For the Link interrupt */ 4987 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); 4988 } 4989 4990 /* 4991 ** ixgbe_sfp_probe - called in the local timer to 4992 ** determine if a port had optics inserted. 4993 */ 4994 static bool ixgbe_sfp_probe(struct adapter *adapter) 4995 { 4996 struct ixgbe_hw *hw = &adapter->hw; 4997 device_t dev = adapter->dev; 4998 bool result = FALSE; 4999 5000 if ((hw->phy.type == ixgbe_phy_nl) && 5001 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 5002 s32 ret = hw->phy.ops.identify_sfp(hw); 5003 if (ret) 5004 goto out; 5005 ret = hw->phy.ops.reset(hw); 5006 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5007 device_printf(dev,"Unsupported SFP+ module detected!"); 5008 device_printf(dev, "Reload driver with supported module.\n"); 5009 adapter->sfp_probe = FALSE; 5010 goto out; 5011 } else 5012 device_printf(dev,"SFP+ module detected!\n"); 5013 /* We now have supported optics */ 5014 adapter->sfp_probe = FALSE; 5015 /* Set the optics type so system reports correctly */ 5016 ixgbe_setup_optics(adapter); 5017 result = TRUE; 5018 } 5019 out: 5020 return (result); 5021 } 5022 5023 /* 5024 ** Tasklet handler for MSIX Link interrupts 5025 ** - do outside interrupt since it might sleep 5026 */ 5027 static void 5028 ixgbe_handle_link(void *context) 5029 { 5030 struct adapter *adapter = context; 5031 5032 ixgbe_check_link(&adapter->hw, 5033 &adapter->link_speed, &adapter->link_up, 0); 5034 ixgbe_update_link_status(adapter); 5035 } 5036 5037 /* 5038 ** Tasklet for handling SFP module interrupts 5039 */ 5040 static void 5041 ixgbe_handle_mod(void *context) 5042 { 5043 struct adapter *adapter = context; 5044 struct ixgbe_hw *hw = &adapter->hw; 5045 device_t dev = adapter->dev; 5046 u32 err; 5047 5048 err = hw->phy.ops.identify_sfp(hw); 5049 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5050 device_printf(dev, 5051 "Unsupported SFP+ module type was detected.\n"); 5052 return; 5053 } 5054 err = hw->mac.ops.setup_sfp(hw); 5055 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5056 device_printf(dev, 5057 "Setup failure - unsupported SFP+ module type.\n"); 5058 return; 5059 } 5060 softint_schedule(adapter->msf_si); 5061 return; 5062 } 5063 5064 5065 /* 5066 ** Tasklet for handling MSF (multispeed fiber) interrupts 5067 */ 5068 static void 5069 ixgbe_handle_msf(void *context) 5070 { 5071 struct adapter *adapter = context; 5072 struct ixgbe_hw *hw = &adapter->hw; 5073 u32 autoneg; 5074 bool negotiate; 5075 5076 autoneg = hw->phy.autoneg_advertised; 5077 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5078 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 5079 if (hw->mac.ops.setup_link) 5080 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); 5081 return; 5082 } 5083 5084 #ifdef IXGBE_FDIR 5085 /* 5086 ** Tasklet for reinitializing the Flow Director filter table 5087 */ 5088 static void 5089 ixgbe_reinit_fdir(void *context) 5090 { 5091 struct adapter *adapter = context; 5092 struct ifnet *ifp = adapter->ifp; 5093 5094 if (adapter->fdir_reinit != 1) /* Shouldn't happen */ 5095 return; 5096 ixgbe_reinit_fdir_tables_82599(&adapter->hw); 5097 adapter->fdir_reinit = 0; 5098 /* Restart the interface */ 5099 ifp->if_flags |= IFF_RUNNING; 5100 return; 5101 } 5102 #endif 5103 5104 /********************************************************************** 5105 * 5106 * Update the board statistics counters. 5107 * 5108 **********************************************************************/ 5109 static void 5110 ixgbe_update_stats_counters(struct adapter *adapter) 5111 { 5112 struct ifnet *ifp = adapter->ifp; 5113 struct ixgbe_hw *hw = &adapter->hw; 5114 u32 missed_rx = 0, bprc, lxon, lxoff, total; 5115 u64 total_missed_rx = 0; 5116 5117 adapter->stats.crcerrs.ev_count += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5118 adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 5119 adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC); 5120 adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC); 5121 5122 for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) { 5123 int j = i % adapter->num_queues; 5124 u32 mp; 5125 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5126 /* missed_rx tallies misses for the gprc workaround */ 5127 missed_rx += mp; 5128 /* global total per queue */ 5129 adapter->stats.mpc[j].ev_count += mp; 5130 /* Running comprehensive total for stats display */ 5131 total_missed_rx += adapter->stats.mpc[j].ev_count; 5132 if (hw->mac.type == ixgbe_mac_82598EB) 5133 adapter->stats.rnbc[j] += 5134 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5135 adapter->stats.pxontxc[j].ev_count += 5136 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5137 adapter->stats.pxonrxc[j].ev_count += 5138 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5139 adapter->stats.pxofftxc[j].ev_count += 5140 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5141 adapter->stats.pxoffrxc[j].ev_count += 5142 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 5143 adapter->stats.pxon2offc[j].ev_count += 5144 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 5145 } 5146 for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) { 5147 int j = i % adapter->num_queues; 5148 adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5149 adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5150 adapter->stats.qbrc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5151 adapter->stats.qbrc[j].ev_count += 5152 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); 5153 adapter->stats.qbtc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5154 adapter->stats.qbtc[j].ev_count += 5155 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); 5156 adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5157 } 5158 adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC); 5159 adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC); 5160 adapter->stats.rlec.ev_count += IXGBE_READ_REG(hw, IXGBE_RLEC); 5161 5162 /* Hardware workaround, gprc counts missed packets */ 5163 adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx; 5164 5165 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5166 adapter->stats.lxontxc.ev_count += lxon; 5167 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5168 adapter->stats.lxofftxc.ev_count += lxoff; 5169 total = lxon + lxoff; 5170 5171 if (hw->mac.type != ixgbe_mac_82598EB) { 5172 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) + 5173 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 5174 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 5175 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN; 5176 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) + 5177 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 5178 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5179 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 5180 } else { 5181 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5182 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 5183 /* 82598 only has a counter in the high register */ 5184 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH); 5185 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN; 5186 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH); 5187 } 5188 5189 /* 5190 * Workaround: mprc hardware is incorrectly counting 5191 * broadcasts, so for now we subtract those. 5192 */ 5193 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5194 adapter->stats.bprc.ev_count += bprc; 5195 adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0); 5196 5197 adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64); 5198 adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127); 5199 adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255); 5200 adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511); 5201 adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5202 adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5203 5204 adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total; 5205 adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total; 5206 adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total; 5207 5208 adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC); 5209 adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC); 5210 adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC); 5211 adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC); 5212 adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 5213 adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 5214 adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 5215 adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR); 5216 adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT); 5217 adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127); 5218 adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255); 5219 adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511); 5220 adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5221 adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5222 adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC); 5223 adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC); 5224 adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5225 adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST); 5226 5227 /* Only read FCOE on 82599 */ 5228 if (hw->mac.type != ixgbe_mac_82598EB) { 5229 adapter->stats.fcoerpdc.ev_count += 5230 IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5231 adapter->stats.fcoeprc.ev_count += 5232 IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5233 adapter->stats.fcoeptc.ev_count += 5234 IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5235 adapter->stats.fcoedwrc.ev_count += 5236 IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5237 adapter->stats.fcoedwtc.ev_count += 5238 IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5239 } 5240 5241 /* Fill out the OS statistics structure */ 5242 ifp->if_ipackets = adapter->stats.gprc.ev_count; 5243 ifp->if_opackets = adapter->stats.gptc.ev_count; 5244 ifp->if_ibytes = adapter->stats.gorc.ev_count; 5245 ifp->if_obytes = adapter->stats.gotc.ev_count; 5246 ifp->if_imcasts = adapter->stats.mprc.ev_count; 5247 ifp->if_collisions = 0; 5248 5249 /* Rx Errors */ 5250 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs.ev_count + 5251 adapter->stats.rlec.ev_count; 5252 } 5253 5254 /** ixgbe_sysctl_tdh_handler - Handler function 5255 * Retrieves the TDH value from the hardware 5256 */ 5257 static int 5258 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 5259 { 5260 struct sysctlnode node; 5261 uint32_t val; 5262 struct tx_ring *txr; 5263 5264 node = *rnode; 5265 txr = (struct tx_ring *)node.sysctl_data; 5266 if (txr == NULL) 5267 return 0; 5268 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me)); 5269 node.sysctl_data = &val; 5270 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5271 } 5272 5273 /** ixgbe_sysctl_tdt_handler - Handler function 5274 * Retrieves the TDT value from the hardware 5275 */ 5276 static int 5277 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 5278 { 5279 struct sysctlnode node; 5280 uint32_t val; 5281 struct tx_ring *txr; 5282 5283 node = *rnode; 5284 txr = (struct tx_ring *)node.sysctl_data; 5285 if (txr == NULL) 5286 return 0; 5287 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me)); 5288 node.sysctl_data = &val; 5289 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5290 } 5291 5292 /** ixgbe_sysctl_rdh_handler - Handler function 5293 * Retrieves the RDH value from the hardware 5294 */ 5295 static int 5296 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 5297 { 5298 struct sysctlnode node; 5299 uint32_t val; 5300 struct rx_ring *rxr; 5301 5302 node = *rnode; 5303 rxr = (struct rx_ring *)node.sysctl_data; 5304 if (rxr == NULL) 5305 return 0; 5306 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me)); 5307 node.sysctl_data = &val; 5308 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5309 } 5310 5311 /** ixgbe_sysctl_rdt_handler - Handler function 5312 * Retrieves the RDT value from the hardware 5313 */ 5314 static int 5315 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 5316 { 5317 struct sysctlnode node; 5318 uint32_t val; 5319 struct rx_ring *rxr; 5320 5321 node = *rnode; 5322 rxr = (struct rx_ring *)node.sysctl_data; 5323 if (rxr == NULL) 5324 return 0; 5325 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me)); 5326 node.sysctl_data = &val; 5327 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5328 } 5329 5330 static int 5331 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 5332 { 5333 struct sysctlnode node; 5334 struct ix_queue *que; 5335 uint32_t reg, usec, rate; 5336 5337 node = *rnode; 5338 que = (struct ix_queue *)node.sysctl_data; 5339 if (que == NULL) 5340 return 0; 5341 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix)); 5342 usec = ((reg & 0x0FF8) >> 3); 5343 if (usec > 0) 5344 rate = 1000000 / usec; 5345 else 5346 rate = 0; 5347 node.sysctl_data = &rate; 5348 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5349 } 5350 5351 const struct sysctlnode * 5352 ixgbe_sysctl_instance(struct adapter *adapter) 5353 { 5354 const char *dvname; 5355 struct sysctllog **log; 5356 int rc; 5357 const struct sysctlnode *rnode; 5358 5359 log = &adapter->sysctllog; 5360 dvname = device_xname(adapter->dev); 5361 5362 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 5363 0, CTLTYPE_NODE, "hw", NULL, 5364 NULL, 0, NULL, 0, CTL_HW, CTL_EOL)) != 0) 5365 goto err; 5366 5367 if ((rc = sysctl_createv(log, 0, &rnode, &rnode, 5368 0, CTLTYPE_NODE, dvname, 5369 SYSCTL_DESCR("ixgbe information and settings"), 5370 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL)) != 0) 5371 goto err; 5372 5373 return rnode; 5374 err: 5375 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 5376 return NULL; 5377 } 5378 5379 /* 5380 * Add sysctl variables, one per statistic, to the system. 5381 */ 5382 static void 5383 ixgbe_add_hw_stats(struct adapter *adapter) 5384 { 5385 device_t dev = adapter->dev; 5386 const struct sysctlnode *rnode, *cnode; 5387 struct sysctllog **log = &adapter->sysctllog; 5388 struct tx_ring *txr = adapter->tx_rings; 5389 struct rx_ring *rxr = adapter->rx_rings; 5390 struct ixgbe_hw *hw = &adapter->hw; 5391 5392 struct ixgbe_hw_stats *stats = &adapter->stats; 5393 5394 /* Driver Statistics */ 5395 #if 0 5396 /* These counters are not updated by the software */ 5397 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5398 CTLFLAG_RD, &adapter->dropped_pkts, 5399 "Driver dropped packets"); 5400 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed", 5401 CTLFLAG_RD, &adapter->mbuf_header_failed, 5402 "???"); 5403 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed", 5404 CTLFLAG_RD, &adapter->mbuf_packet_failed, 5405 "???"); 5406 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail", 5407 CTLFLAG_RD, &adapter->no_tx_map_avail, 5408 "???"); 5409 #endif 5410 evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC, 5411 NULL, device_xname(dev), "Handled queue in softint"); 5412 evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC, 5413 NULL, device_xname(dev), "Requeued in softint"); 5414 evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC, 5415 NULL, device_xname(dev), "Interrupt handler more rx"); 5416 evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC, 5417 NULL, device_xname(dev), "Interrupt handler more tx"); 5418 evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC, 5419 NULL, device_xname(dev), "Interrupt handler tx loops"); 5420 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 5421 NULL, device_xname(dev), "Driver tx dma soft fail EFBIG"); 5422 evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC, 5423 NULL, device_xname(dev), "m_defrag() failed"); 5424 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 5425 NULL, device_xname(dev), "Driver tx dma hard fail EFBIG"); 5426 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 5427 NULL, device_xname(dev), "Driver tx dma hard fail EINVAL"); 5428 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 5429 NULL, device_xname(dev), "Driver tx dma hard fail other"); 5430 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 5431 NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN"); 5432 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 5433 NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM"); 5434 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 5435 NULL, device_xname(dev), "Watchdog timeouts"); 5436 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 5437 NULL, device_xname(dev), "TSO errors"); 5438 evcnt_attach_dynamic(&adapter->tso_tx, EVCNT_TYPE_MISC, 5439 NULL, device_xname(dev), "TSO"); 5440 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC, 5441 NULL, device_xname(dev), "Link MSIX IRQ Handled"); 5442 5443 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5444 snprintf(adapter->queues[i].evnamebuf, 5445 sizeof(adapter->queues[i].evnamebuf), "%s queue%d", 5446 device_xname(dev), i); 5447 snprintf(adapter->queues[i].namebuf, 5448 sizeof(adapter->queues[i].namebuf), "queue%d", i); 5449 5450 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5451 aprint_error_dev(dev, "could not create sysctl root\n"); 5452 break; 5453 } 5454 5455 if (sysctl_createv(log, 0, &rnode, &rnode, 5456 0, CTLTYPE_NODE, 5457 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 5458 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 5459 break; 5460 5461 if (sysctl_createv(log, 0, &rnode, &cnode, 5462 CTLFLAG_READONLY, CTLTYPE_INT, 5463 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 5464 ixgbe_sysctl_interrupt_rate_handler, 0, 5465 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 5466 break; 5467 5468 if (sysctl_createv(log, 0, &rnode, &cnode, 5469 CTLFLAG_READONLY, CTLTYPE_INT, 5470 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 5471 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 5472 0, CTL_CREATE, CTL_EOL) != 0) 5473 break; 5474 5475 if (sysctl_createv(log, 0, &rnode, &cnode, 5476 CTLFLAG_READONLY, CTLTYPE_INT, 5477 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 5478 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 5479 0, CTL_CREATE, CTL_EOL) != 0) 5480 break; 5481 5482 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 5483 NULL, adapter->queues[i].evnamebuf, 5484 "Queue No Descriptor Available"); 5485 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 5486 NULL, adapter->queues[i].evnamebuf, 5487 "Queue Packets Transmitted"); 5488 5489 #ifdef LRO 5490 struct lro_ctrl *lro = &rxr->lro; 5491 #endif /* LRO */ 5492 5493 if (sysctl_createv(log, 0, &rnode, &cnode, 5494 CTLFLAG_READONLY, 5495 CTLTYPE_INT, 5496 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 5497 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 5498 CTL_CREATE, CTL_EOL) != 0) 5499 break; 5500 5501 if (sysctl_createv(log, 0, &rnode, &cnode, 5502 CTLFLAG_READONLY, 5503 CTLTYPE_INT, 5504 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 5505 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 5506 CTL_CREATE, CTL_EOL) != 0) 5507 break; 5508 5509 if (i < __arraycount(adapter->stats.mpc)) { 5510 evcnt_attach_dynamic(&adapter->stats.mpc[i], 5511 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5512 "Missed Packet Count"); 5513 } 5514 if (i < __arraycount(adapter->stats.pxontxc)) { 5515 evcnt_attach_dynamic(&adapter->stats.pxontxc[i], 5516 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5517 "pxontxc"); 5518 evcnt_attach_dynamic(&adapter->stats.pxonrxc[i], 5519 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5520 "pxonrxc"); 5521 evcnt_attach_dynamic(&adapter->stats.pxofftxc[i], 5522 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5523 "pxofftxc"); 5524 evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i], 5525 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5526 "pxoffrxc"); 5527 evcnt_attach_dynamic(&adapter->stats.pxon2offc[i], 5528 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5529 "pxon2offc"); 5530 } 5531 if (i < __arraycount(adapter->stats.qprc)) { 5532 evcnt_attach_dynamic(&adapter->stats.qprc[i], 5533 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5534 "qprc"); 5535 evcnt_attach_dynamic(&adapter->stats.qptc[i], 5536 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5537 "qptc"); 5538 evcnt_attach_dynamic(&adapter->stats.qbrc[i], 5539 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5540 "qbrc"); 5541 evcnt_attach_dynamic(&adapter->stats.qbtc[i], 5542 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5543 "qbtc"); 5544 evcnt_attach_dynamic(&adapter->stats.qprdc[i], 5545 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5546 "qprdc"); 5547 } 5548 5549 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 5550 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 5551 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 5552 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 5553 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 5554 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 5555 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 5556 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 5557 evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC, 5558 NULL, adapter->queues[i].evnamebuf, "Rx split packets"); 5559 evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC, 5560 NULL, adapter->queues[i].evnamebuf, "Rx interrupts"); 5561 #ifdef LRO 5562 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 5563 CTLFLAG_RD, &lro->lro_queued, 0, 5564 "LRO Queued"); 5565 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 5566 CTLFLAG_RD, &lro->lro_flushed, 0, 5567 "LRO Flushed"); 5568 #endif /* LRO */ 5569 } 5570 5571 /* MAC stats get the own sub node */ 5572 5573 5574 snprintf(stats->namebuf, 5575 sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev)); 5576 5577 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 5578 stats->namebuf, "rx csum offload - IP"); 5579 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 5580 stats->namebuf, "rx csum offload - L4"); 5581 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 5582 stats->namebuf, "rx csum offload - IP bad"); 5583 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 5584 stats->namebuf, "rx csum offload - L4 bad"); 5585 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 5586 stats->namebuf, "Interrupt conditions zero"); 5587 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 5588 stats->namebuf, "Legacy interrupts"); 5589 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 5590 stats->namebuf, "CRC Errors"); 5591 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 5592 stats->namebuf, "Illegal Byte Errors"); 5593 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 5594 stats->namebuf, "Byte Errors"); 5595 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 5596 stats->namebuf, "MAC Short Packets Discarded"); 5597 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 5598 stats->namebuf, "MAC Local Faults"); 5599 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 5600 stats->namebuf, "MAC Remote Faults"); 5601 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 5602 stats->namebuf, "Receive Length Errors"); 5603 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 5604 stats->namebuf, "Link XON Transmitted"); 5605 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 5606 stats->namebuf, "Link XON Received"); 5607 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 5608 stats->namebuf, "Link XOFF Transmitted"); 5609 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 5610 stats->namebuf, "Link XOFF Received"); 5611 5612 /* Packet Reception Stats */ 5613 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 5614 stats->namebuf, "Total Octets Received"); 5615 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 5616 stats->namebuf, "Good Octets Received"); 5617 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 5618 stats->namebuf, "Total Packets Received"); 5619 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 5620 stats->namebuf, "Good Packets Received"); 5621 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 5622 stats->namebuf, "Multicast Packets Received"); 5623 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 5624 stats->namebuf, "Broadcast Packets Received"); 5625 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 5626 stats->namebuf, "64 byte frames received "); 5627 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 5628 stats->namebuf, "65-127 byte frames received"); 5629 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 5630 stats->namebuf, "128-255 byte frames received"); 5631 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 5632 stats->namebuf, "256-511 byte frames received"); 5633 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 5634 stats->namebuf, "512-1023 byte frames received"); 5635 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 5636 stats->namebuf, "1023-1522 byte frames received"); 5637 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 5638 stats->namebuf, "Receive Undersized"); 5639 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 5640 stats->namebuf, "Fragmented Packets Received "); 5641 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 5642 stats->namebuf, "Oversized Packets Received"); 5643 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 5644 stats->namebuf, "Received Jabber"); 5645 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 5646 stats->namebuf, "Management Packets Received"); 5647 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 5648 stats->namebuf, "Management Packets Dropped"); 5649 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 5650 stats->namebuf, "Checksum Errors"); 5651 5652 /* Packet Transmission Stats */ 5653 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 5654 stats->namebuf, "Good Octets Transmitted"); 5655 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 5656 stats->namebuf, "Total Packets Transmitted"); 5657 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 5658 stats->namebuf, "Good Packets Transmitted"); 5659 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 5660 stats->namebuf, "Broadcast Packets Transmitted"); 5661 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 5662 stats->namebuf, "Multicast Packets Transmitted"); 5663 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 5664 stats->namebuf, "Management Packets Transmitted"); 5665 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 5666 stats->namebuf, "64 byte frames transmitted "); 5667 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 5668 stats->namebuf, "65-127 byte frames transmitted"); 5669 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 5670 stats->namebuf, "128-255 byte frames transmitted"); 5671 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 5672 stats->namebuf, "256-511 byte frames transmitted"); 5673 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 5674 stats->namebuf, "512-1023 byte frames transmitted"); 5675 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 5676 stats->namebuf, "1024-1522 byte frames transmitted"); 5677 5678 /* FC Stats */ 5679 evcnt_attach_dynamic(&stats->fccrc, EVCNT_TYPE_MISC, NULL, 5680 stats->namebuf, "FC CRC Errors"); 5681 evcnt_attach_dynamic(&stats->fclast, EVCNT_TYPE_MISC, NULL, 5682 stats->namebuf, "FC Last Error"); 5683 if (hw->mac.type != ixgbe_mac_82598EB) { 5684 evcnt_attach_dynamic(&stats->fcoerpdc, EVCNT_TYPE_MISC, NULL, 5685 stats->namebuf, "FCoE Packets Dropped"); 5686 evcnt_attach_dynamic(&stats->fcoeprc, EVCNT_TYPE_MISC, NULL, 5687 stats->namebuf, "FCoE Packets Received"); 5688 evcnt_attach_dynamic(&stats->fcoeptc, EVCNT_TYPE_MISC, NULL, 5689 stats->namebuf, "FCoE Packets Transmitted"); 5690 evcnt_attach_dynamic(&stats->fcoedwrc, EVCNT_TYPE_MISC, NULL, 5691 stats->namebuf, "FCoE DWords Received"); 5692 evcnt_attach_dynamic(&stats->fcoedwtc, EVCNT_TYPE_MISC, NULL, 5693 stats->namebuf, "FCoE DWords Transmitted"); 5694 } 5695 } 5696 5697 /* 5698 ** Set flow control using sysctl: 5699 ** Flow control values: 5700 ** 0 - off 5701 ** 1 - rx pause 5702 ** 2 - tx pause 5703 ** 3 - full 5704 */ 5705 static int 5706 ixgbe_set_flowcntl(SYSCTLFN_ARGS) 5707 { 5708 struct sysctlnode node; 5709 int error; 5710 int last = ixgbe_flow_control; 5711 struct adapter *adapter; 5712 5713 node = *rnode; 5714 adapter = (struct adapter *)node.sysctl_data; 5715 node.sysctl_data = &ixgbe_flow_control; 5716 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5717 if (error != 0 || newp == NULL) 5718 return error; 5719 5720 /* Don't bother if it's not changed */ 5721 if (ixgbe_flow_control == last) 5722 return (0); 5723 5724 switch (ixgbe_flow_control) { 5725 case ixgbe_fc_rx_pause: 5726 case ixgbe_fc_tx_pause: 5727 case ixgbe_fc_full: 5728 adapter->hw.fc.requested_mode = ixgbe_flow_control; 5729 break; 5730 case ixgbe_fc_none: 5731 default: 5732 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5733 } 5734 5735 ixgbe_fc_enable(&adapter->hw, 0); 5736 return 0; 5737 } 5738 5739 static void 5740 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 5741 const char *description, int *limit, int value) 5742 { 5743 const struct sysctlnode *rnode, *cnode; 5744 struct sysctllog **log = &adapter->sysctllog; 5745 5746 *limit = value; 5747 5748 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) 5749 aprint_error_dev(adapter->dev, 5750 "could not create sysctl root\n"); 5751 else if (sysctl_createv(log, 0, &rnode, &cnode, 5752 CTLFLAG_READWRITE, 5753 CTLTYPE_INT, 5754 name, SYSCTL_DESCR(description), 5755 NULL, 0, limit, 0, 5756 CTL_CREATE, CTL_EOL) != 0) { 5757 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 5758 __func__); 5759 } 5760 } 5761 5762 /* 5763 ** Control link advertise speed: 5764 ** 0 - normal 5765 ** 1 - advertise only 1G 5766 */ 5767 static int 5768 ixgbe_set_advertise(SYSCTLFN_ARGS) 5769 { 5770 struct sysctlnode node; 5771 int t, error; 5772 struct adapter *adapter; 5773 struct ixgbe_hw *hw; 5774 ixgbe_link_speed speed, last; 5775 5776 node = *rnode; 5777 adapter = (struct adapter *)node.sysctl_data; 5778 t = adapter->advertise; 5779 node.sysctl_data = &t; 5780 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5781 if (error != 0 || newp == NULL) 5782 return error; 5783 5784 if (t == -1) 5785 return 0; 5786 5787 adapter->advertise = t; 5788 5789 hw = &adapter->hw; 5790 last = hw->phy.autoneg_advertised; 5791 5792 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5793 (hw->phy.multispeed_fiber))) 5794 return 0; 5795 5796 if (adapter->advertise == 1) 5797 speed = IXGBE_LINK_SPEED_1GB_FULL; 5798 else 5799 speed = IXGBE_LINK_SPEED_1GB_FULL | 5800 IXGBE_LINK_SPEED_10GB_FULL; 5801 5802 if (speed == last) /* no change */ 5803 return 0; 5804 5805 hw->mac.autotry_restart = TRUE; 5806 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); 5807 5808 return 0; 5809 } 5810