1 /****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD: src/sys/dev/ixgbe/ixv.c,v 1.2 2011/03/23 13:10:15 jhb Exp $*/ 34 /*$NetBSD: ixv.c,v 1.2 2014/03/25 16:19:14 christos Exp $*/ 35 36 #include "opt_inet.h" 37 38 #include "ixv.h" 39 40 /********************************************************************* 41 * Driver version 42 *********************************************************************/ 43 char ixv_driver_version[] = "1.0.0"; 44 45 /********************************************************************* 46 * PCI Device ID Table 47 * 48 * Used by probe to select devices to load on 49 * Last field stores an index into ixv_strings 50 * Last entry must be all 0s 51 * 52 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 53 *********************************************************************/ 54 55 static ixv_vendor_info_t ixv_vendor_info_array[] = 56 { 57 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 58 /* required last entry */ 59 {0, 0, 0, 0, 0} 60 }; 61 62 /********************************************************************* 63 * Table of branding strings 64 *********************************************************************/ 65 66 static char *ixv_strings[] = { 67 "Intel(R) PRO/10GbE Virtual Function Network Driver" 68 }; 69 70 /********************************************************************* 71 * Function prototypes 72 *********************************************************************/ 73 static int ixv_probe(device_t); 74 static int ixv_attach(device_t); 75 static int ixv_detach(device_t); 76 static int ixv_shutdown(device_t); 77 #if __FreeBSD_version < 800000 78 static void ixv_start(struct ifnet *); 79 static void ixv_start_locked(struct tx_ring *, struct ifnet *); 80 #else 81 static int ixv_mq_start(struct ifnet *, struct mbuf *); 82 static int ixv_mq_start_locked(struct ifnet *, 83 struct tx_ring *, struct mbuf *); 84 static void ixv_qflush(struct ifnet *); 85 #endif 86 static int ixv_ioctl(struct ifnet *, u_long, caddr_t); 87 static void ixv_init(void *); 88 static void ixv_init_locked(struct adapter *); 89 static void ixv_stop(void *); 90 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 91 static int ixv_media_change(struct ifnet *); 92 static void ixv_identify_hardware(struct adapter *); 93 static int ixv_allocate_pci_resources(struct adapter *); 94 static int ixv_allocate_msix(struct adapter *); 95 static int ixv_allocate_queues(struct adapter *); 96 static int ixv_setup_msix(struct adapter *); 97 static void ixv_free_pci_resources(struct adapter *); 98 static void ixv_local_timer(void *); 99 static void ixv_setup_interface(device_t, struct adapter *); 100 static void ixv_config_link(struct adapter *); 101 102 static int ixv_allocate_transmit_buffers(struct tx_ring *); 103 static int ixv_setup_transmit_structures(struct adapter *); 104 static void ixv_setup_transmit_ring(struct tx_ring *); 105 static void ixv_initialize_transmit_units(struct adapter *); 106 static void ixv_free_transmit_structures(struct adapter *); 107 static void ixv_free_transmit_buffers(struct tx_ring *); 108 109 static int ixv_allocate_receive_buffers(struct rx_ring *); 110 static int ixv_setup_receive_structures(struct adapter *); 111 static int ixv_setup_receive_ring(struct rx_ring *); 112 static void ixv_initialize_receive_units(struct adapter *); 113 static void ixv_free_receive_structures(struct adapter *); 114 static void ixv_free_receive_buffers(struct rx_ring *); 115 116 static void ixv_enable_intr(struct adapter *); 117 static void ixv_disable_intr(struct adapter *); 118 static bool ixv_txeof(struct tx_ring *); 119 static bool ixv_rxeof(struct ix_queue *, int); 120 static void ixv_rx_checksum(u32, struct mbuf *, u32); 121 static void ixv_set_multi(struct adapter *); 122 static void ixv_update_link_status(struct adapter *); 123 static void ixv_refresh_mbufs(struct rx_ring *, int); 124 static int ixv_xmit(struct tx_ring *, struct mbuf **); 125 static int ixv_sysctl_stats(SYSCTL_HANDLER_ARGS); 126 static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 127 static int ixv_set_flowcntl(SYSCTL_HANDLER_ARGS); 128 static int ixv_dma_malloc(struct adapter *, bus_size_t, 129 struct ixv_dma_alloc *, int); 130 static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 131 static void ixv_add_rx_process_limit(struct adapter *, const char *, 132 const char *, int *, int); 133 static bool ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 134 static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 135 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 136 static void ixv_configure_ivars(struct adapter *); 137 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 138 139 static void ixv_setup_vlan_support(struct adapter *); 140 static void ixv_register_vlan(void *, struct ifnet *, u16); 141 static void ixv_unregister_vlan(void *, struct ifnet *, u16); 142 143 static void ixv_save_stats(struct adapter *); 144 static void ixv_init_stats(struct adapter *); 145 static void ixv_update_stats(struct adapter *); 146 147 static __inline void ixv_rx_discard(struct rx_ring *, int); 148 static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 149 struct mbuf *, u32); 150 151 /* The MSI/X Interrupt handlers */ 152 static void ixv_msix_que(void *); 153 static void ixv_msix_mbx(void *); 154 155 /* Deferred interrupt tasklets */ 156 static void ixv_handle_que(void *, int); 157 static void ixv_handle_mbx(void *, int); 158 159 /********************************************************************* 160 * FreeBSD Device Interface Entry Points 161 *********************************************************************/ 162 163 static device_method_t ixv_methods[] = { 164 /* Device interface */ 165 DEVMETHOD(device_probe, ixv_probe), 166 DEVMETHOD(device_attach, ixv_attach), 167 DEVMETHOD(device_detach, ixv_detach), 168 DEVMETHOD(device_shutdown, ixv_shutdown), 169 {0, 0} 170 }; 171 172 #if 0 173 static driver_t ixv_driver = { 174 "ix", ixv_methods, sizeof(struct adapter), 175 }; 176 177 extern devclass_t ixgbe_devclass; 178 DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 179 MODULE_DEPEND(ixv, pci, 1, 1, 1); 180 MODULE_DEPEND(ixv, ether, 1, 1, 1); 181 #endif 182 183 /* 184 ** TUNEABLE PARAMETERS: 185 */ 186 187 /* 188 ** AIM: Adaptive Interrupt Moderation 189 ** which means that the interrupt rate 190 ** is varied over time based on the 191 ** traffic for that interrupt vector 192 */ 193 static int ixv_enable_aim = FALSE; 194 #define TUNABLE_INT(__x, __y) 195 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 196 197 /* How many packets rxeof tries to clean at a time */ 198 static int ixv_rx_process_limit = 128; 199 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 200 201 /* Flow control setting, default to full */ 202 static int ixv_flow_control = ixgbe_fc_full; 203 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 204 205 /* 206 * Header split: this causes the hardware to DMA 207 * the header into a seperate mbuf from the payload, 208 * it can be a performance win in some workloads, but 209 * in others it actually hurts, its off by default. 210 */ 211 static bool ixv_header_split = FALSE; 212 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 213 214 /* 215 ** Number of TX descriptors per ring, 216 ** setting higher than RX as this seems 217 ** the better performing choice. 218 */ 219 static int ixv_txd = DEFAULT_TXD; 220 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 221 222 /* Number of RX descriptors per ring */ 223 static int ixv_rxd = DEFAULT_RXD; 224 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 225 226 /* 227 ** Shadow VFTA table, this is needed because 228 ** the real filter table gets cleared during 229 ** a soft reset and we need to repopulate it. 230 */ 231 static u32 ixv_shadow_vfta[VFTA_SIZE]; 232 233 /********************************************************************* 234 * Device identification routine 235 * 236 * ixv_probe determines if the driver should be loaded on 237 * adapter based on PCI vendor/device id of the adapter. 238 * 239 * return 0 on success, positive on failure 240 *********************************************************************/ 241 242 static int 243 ixv_probe(device_t dev) 244 { 245 ixv_vendor_info_t *ent; 246 247 u16 pci_vendor_id = 0; 248 u16 pci_device_id = 0; 249 u16 pci_subvendor_id = 0; 250 u16 pci_subdevice_id = 0; 251 char adapter_name[256]; 252 253 254 pci_vendor_id = pci_get_vendor(dev); 255 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 256 return (ENXIO); 257 258 pci_device_id = pci_get_device(dev); 259 pci_subvendor_id = pci_get_subvendor(dev); 260 pci_subdevice_id = pci_get_subdevice(dev); 261 262 ent = ixv_vendor_info_array; 263 while (ent->vendor_id != 0) { 264 if ((pci_vendor_id == ent->vendor_id) && 265 (pci_device_id == ent->device_id) && 266 267 ((pci_subvendor_id == ent->subvendor_id) || 268 (ent->subvendor_id == 0)) && 269 270 ((pci_subdevice_id == ent->subdevice_id) || 271 (ent->subdevice_id == 0))) { 272 snprintf(adapter_name, sizeof(adapter_name), 273 "%s, Version - %s", ixv_strings[ent->index], 274 ixv_driver_version); 275 device_set_desc_copy(dev, adapter_name); 276 return (0); 277 } 278 ent++; 279 } 280 return (ENXIO); 281 } 282 283 /********************************************************************* 284 * Device initialization routine 285 * 286 * The attach entry point is called when the driver is being loaded. 287 * This routine identifies the type of hardware, allocates all resources 288 * and initializes the hardware. 289 * 290 * return 0 on success, positive on failure 291 *********************************************************************/ 292 293 static int 294 ixv_attach(device_t dev) 295 { 296 struct adapter *adapter; 297 struct ixgbe_hw *hw; 298 int error = 0; 299 300 INIT_DEBUGOUT("ixv_attach: begin"); 301 302 /* Allocate, clear, and link in our adapter structure */ 303 adapter = device_get_softc(dev); 304 adapter->dev = adapter->osdep.dev = dev; 305 hw = &adapter->hw; 306 307 /* Core Lock Init*/ 308 IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 309 310 /* SYSCTL APIs */ 311 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 312 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 313 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 314 adapter, 0, ixv_sysctl_stats, "I", "Statistics"); 315 316 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 317 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 318 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 319 adapter, 0, ixv_sysctl_debug, "I", "Debug Info"); 320 321 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 322 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 323 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 324 adapter, 0, ixv_set_flowcntl, "I", "Flow Control"); 325 326 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 327 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 328 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 329 &ixv_enable_aim, 1, "Interrupt Moderation"); 330 331 /* Set up the timer callout */ 332 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 333 334 /* Determine hardware revision */ 335 ixv_identify_hardware(adapter); 336 337 /* Do base PCI setup - map BAR0 */ 338 if (ixv_allocate_pci_resources(adapter)) { 339 device_printf(dev, "Allocation of PCI resources failed\n"); 340 error = ENXIO; 341 goto err_out; 342 } 343 344 /* Do descriptor calc and sanity checks */ 345 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 346 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 347 device_printf(dev, "TXD config issue, using default!\n"); 348 adapter->num_tx_desc = DEFAULT_TXD; 349 } else 350 adapter->num_tx_desc = ixv_txd; 351 352 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 353 ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) { 354 device_printf(dev, "RXD config issue, using default!\n"); 355 adapter->num_rx_desc = DEFAULT_RXD; 356 } else 357 adapter->num_rx_desc = ixv_rxd; 358 359 /* Allocate our TX/RX Queues */ 360 if (ixv_allocate_queues(adapter)) { 361 error = ENOMEM; 362 goto err_out; 363 } 364 365 /* 366 ** Initialize the shared code: its 367 ** at this point the mac type is set. 368 */ 369 error = ixgbe_init_shared_code(hw); 370 if (error) { 371 device_printf(dev,"Shared Code Initialization Failure\n"); 372 error = EIO; 373 goto err_late; 374 } 375 376 /* Setup the mailbox */ 377 ixgbe_init_mbx_params_vf(hw); 378 379 ixgbe_reset_hw(hw); 380 381 /* Get Hardware Flow Control setting */ 382 hw->fc.requested_mode = ixgbe_fc_full; 383 hw->fc.pause_time = IXV_FC_PAUSE; 384 hw->fc.low_water = IXV_FC_LO; 385 hw->fc.high_water = IXV_FC_HI; 386 hw->fc.send_xon = TRUE; 387 388 error = ixgbe_init_hw(hw); 389 if (error) { 390 device_printf(dev,"Hardware Initialization Failure\n"); 391 error = EIO; 392 goto err_late; 393 } 394 395 error = ixv_allocate_msix(adapter); 396 if (error) 397 goto err_late; 398 399 /* Setup OS specific network interface */ 400 ixv_setup_interface(dev, adapter); 401 402 /* Sysctl for limiting the amount of work done in the taskqueue */ 403 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 404 "max number of rx packets to process", &adapter->rx_process_limit, 405 ixv_rx_process_limit); 406 407 /* Do the stats setup */ 408 ixv_save_stats(adapter); 409 ixv_init_stats(adapter); 410 411 /* Register for VLAN events */ 412 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 413 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 414 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 415 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 416 417 INIT_DEBUGOUT("ixv_attach: end"); 418 return (0); 419 420 err_late: 421 ixv_free_transmit_structures(adapter); 422 ixv_free_receive_structures(adapter); 423 err_out: 424 ixv_free_pci_resources(adapter); 425 return (error); 426 427 } 428 429 /********************************************************************* 430 * Device removal routine 431 * 432 * The detach entry point is called when the driver is being removed. 433 * This routine stops the adapter and deallocates all the resources 434 * that were allocated for driver operation. 435 * 436 * return 0 on success, positive on failure 437 *********************************************************************/ 438 439 static int 440 ixv_detach(device_t dev) 441 { 442 struct adapter *adapter = device_get_softc(dev); 443 struct ix_queue *que = adapter->queues; 444 445 INIT_DEBUGOUT("ixv_detach: begin"); 446 447 /* Make sure VLANS are not using driver */ 448 if (adapter->ifp->if_vlantrunk != NULL) { 449 device_printf(dev,"Vlan in use, detach first\n"); 450 return (EBUSY); 451 } 452 453 IXV_CORE_LOCK(adapter); 454 ixv_stop(adapter); 455 IXV_CORE_UNLOCK(adapter); 456 457 for (int i = 0; i < adapter->num_queues; i++, que++) { 458 if (que->tq) { 459 taskqueue_drain(que->tq, &que->que_task); 460 taskqueue_free(que->tq); 461 } 462 } 463 464 /* Drain the Link queue */ 465 if (adapter->tq) { 466 taskqueue_drain(adapter->tq, &adapter->mbx_task); 467 taskqueue_free(adapter->tq); 468 } 469 470 /* Unregister VLAN events */ 471 if (adapter->vlan_attach != NULL) 472 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 473 if (adapter->vlan_detach != NULL) 474 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 475 476 ether_ifdetach(adapter->ifp); 477 callout_drain(&adapter->timer); 478 ixv_free_pci_resources(adapter); 479 bus_generic_detach(dev); 480 if_free(adapter->ifp); 481 482 ixv_free_transmit_structures(adapter); 483 ixv_free_receive_structures(adapter); 484 485 IXV_CORE_LOCK_DESTROY(adapter); 486 return (0); 487 } 488 489 /********************************************************************* 490 * 491 * Shutdown entry point 492 * 493 **********************************************************************/ 494 static int 495 ixv_shutdown(device_t dev) 496 { 497 struct adapter *adapter = device_get_softc(dev); 498 IXV_CORE_LOCK(adapter); 499 ixv_stop(adapter); 500 IXV_CORE_UNLOCK(adapter); 501 return (0); 502 } 503 504 #if __FreeBSD_version < 800000 505 /********************************************************************* 506 * Transmit entry point 507 * 508 * ixv_start is called by the stack to initiate a transmit. 509 * The driver will remain in this routine as long as there are 510 * packets to transmit and transmit resources are available. 511 * In case resources are not available stack is notified and 512 * the packet is requeued. 513 **********************************************************************/ 514 static void 515 ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 516 { 517 struct mbuf *m_head; 518 struct adapter *adapter = txr->adapter; 519 520 IXV_TX_LOCK_ASSERT(txr); 521 522 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 523 IFF_DRV_RUNNING) 524 return; 525 if (!adapter->link_active) 526 return; 527 528 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 529 530 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 531 if (m_head == NULL) 532 break; 533 534 if (ixv_xmit(txr, &m_head)) { 535 if (m_head == NULL) 536 break; 537 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 538 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 539 break; 540 } 541 /* Send a copy of the frame to the BPF listener */ 542 ETHER_BPF_MTAP(ifp, m_head); 543 544 /* Set watchdog on */ 545 txr->watchdog_check = TRUE; 546 txr->watchdog_time = ticks; 547 548 } 549 return; 550 } 551 552 /* 553 * Legacy TX start - called by the stack, this 554 * always uses the first tx ring, and should 555 * not be used with multiqueue tx enabled. 556 */ 557 static void 558 ixv_start(struct ifnet *ifp) 559 { 560 struct adapter *adapter = ifp->if_softc; 561 struct tx_ring *txr = adapter->tx_rings; 562 563 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 564 IXV_TX_LOCK(txr); 565 ixv_start_locked(txr, ifp); 566 IXV_TX_UNLOCK(txr); 567 } 568 return; 569 } 570 571 #else 572 573 /* 574 ** Multiqueue Transmit driver 575 ** 576 */ 577 static int 578 ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 579 { 580 struct adapter *adapter = ifp->if_softc; 581 struct ix_queue *que; 582 struct tx_ring *txr; 583 int i = 0, err = 0; 584 585 /* Which queue to use */ 586 if ((m->m_flags & M_FLOWID) != 0) 587 i = m->m_pkthdr.flowid % adapter->num_queues; 588 589 txr = &adapter->tx_rings[i]; 590 que = &adapter->queues[i]; 591 592 if (IXV_TX_TRYLOCK(txr)) { 593 err = ixv_mq_start_locked(ifp, txr, m); 594 IXV_TX_UNLOCK(txr); 595 } else { 596 err = drbr_enqueue(ifp, txr->br, m); 597 taskqueue_enqueue(que->tq, &que->que_task); 598 } 599 600 return (err); 601 } 602 603 static int 604 ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 605 { 606 struct adapter *adapter = txr->adapter; 607 struct mbuf *next; 608 int enqueued, err = 0; 609 610 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 611 IFF_DRV_RUNNING || adapter->link_active == 0) { 612 if (m != NULL) 613 err = drbr_enqueue(ifp, txr->br, m); 614 return (err); 615 } 616 617 /* Do a clean if descriptors are low */ 618 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 619 ixv_txeof(txr); 620 621 enqueued = 0; 622 if (m == NULL) { 623 next = drbr_dequeue(ifp, txr->br); 624 } else if (drbr_needs_enqueue(ifp, txr->br)) { 625 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 626 return (err); 627 next = drbr_dequeue(ifp, txr->br); 628 } else 629 next = m; 630 631 /* Process the queue */ 632 while (next != NULL) { 633 if ((err = ixv_xmit(txr, &next)) != 0) { 634 if (next != NULL) 635 err = drbr_enqueue(ifp, txr->br, next); 636 break; 637 } 638 enqueued++; 639 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 640 /* Send a copy of the frame to the BPF listener */ 641 ETHER_BPF_MTAP(ifp, next); 642 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 643 break; 644 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 645 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 646 break; 647 } 648 next = drbr_dequeue(ifp, txr->br); 649 } 650 651 if (enqueued > 0) { 652 /* Set watchdog on */ 653 txr->watchdog_check = TRUE; 654 txr->watchdog_time = ticks; 655 } 656 657 return (err); 658 } 659 660 /* 661 ** Flush all ring buffers 662 */ 663 static void 664 ixv_qflush(struct ifnet *ifp) 665 { 666 struct adapter *adapter = ifp->if_softc; 667 struct tx_ring *txr = adapter->tx_rings; 668 struct mbuf *m; 669 670 for (int i = 0; i < adapter->num_queues; i++, txr++) { 671 IXV_TX_LOCK(txr); 672 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 673 m_freem(m); 674 IXV_TX_UNLOCK(txr); 675 } 676 if_qflush(ifp); 677 } 678 679 #endif 680 681 /********************************************************************* 682 * Ioctl entry point 683 * 684 * ixv_ioctl is called when the user wants to configure the 685 * interface. 686 * 687 * return 0 on success, positive on failure 688 **********************************************************************/ 689 690 static int 691 ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 692 { 693 struct adapter *adapter = ifp->if_softc; 694 struct ifreq *ifr = (struct ifreq *) data; 695 int error = 0; 696 697 switch (command) { 698 699 case SIOCSIFMTU: 700 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 701 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 702 error = EINVAL; 703 } else { 704 IXV_CORE_LOCK(adapter); 705 ifp->if_mtu = ifr->ifr_mtu; 706 adapter->max_frame_size = 707 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 708 ixv_init_locked(adapter); 709 IXV_CORE_UNLOCK(adapter); 710 } 711 break; 712 case SIOCSIFFLAGS: 713 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 714 IXV_CORE_LOCK(adapter); 715 if (ifp->if_flags & IFF_UP) { 716 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 717 ixv_init_locked(adapter); 718 } else 719 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 720 ixv_stop(adapter); 721 adapter->if_flags = ifp->if_flags; 722 IXV_CORE_UNLOCK(adapter); 723 break; 724 case SIOCADDMULTI: 725 case SIOCDELMULTI: 726 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 727 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 728 IXV_CORE_LOCK(adapter); 729 ixv_disable_intr(adapter); 730 ixv_set_multi(adapter); 731 ixv_enable_intr(adapter); 732 IXV_CORE_UNLOCK(adapter); 733 } 734 break; 735 case SIOCSIFMEDIA: 736 case SIOCGIFMEDIA: 737 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 738 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 739 break; 740 case SIOCSIFCAP: 741 { 742 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 743 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 744 if (mask & IFCAP_HWCSUM) 745 ifp->if_capenable ^= IFCAP_HWCSUM; 746 if (mask & IFCAP_TSO4) 747 ifp->if_capenable ^= IFCAP_TSO4; 748 if (mask & IFCAP_LRO) 749 ifp->if_capenable ^= IFCAP_LRO; 750 if (mask & IFCAP_VLAN_HWTAGGING) 751 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 752 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 753 IXV_CORE_LOCK(adapter); 754 ixv_init_locked(adapter); 755 IXV_CORE_UNLOCK(adapter); 756 } 757 VLAN_CAPABILITIES(ifp); 758 break; 759 } 760 761 default: 762 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 763 error = ether_ioctl(ifp, command, data); 764 break; 765 } 766 767 return (error); 768 } 769 770 /********************************************************************* 771 * Init entry point 772 * 773 * This routine is used in two ways. It is used by the stack as 774 * init entry point in network interface structure. It is also used 775 * by the driver as a hw/sw initialization routine to get to a 776 * consistent state. 777 * 778 * return 0 on success, positive on failure 779 **********************************************************************/ 780 #define IXGBE_MHADD_MFS_SHIFT 16 781 782 static void 783 ixv_init_locked(struct adapter *adapter) 784 { 785 struct ifnet *ifp = adapter->ifp; 786 device_t dev = adapter->dev; 787 struct ixgbe_hw *hw = &adapter->hw; 788 u32 mhadd, gpie; 789 790 INIT_DEBUGOUT("ixv_init: begin"); 791 mtx_assert(&adapter->core_mtx, MA_OWNED); 792 hw->adapter_stopped = FALSE; 793 ixgbe_stop_adapter(hw); 794 callout_stop(&adapter->timer); 795 796 /* reprogram the RAR[0] in case user changed it. */ 797 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 798 799 /* Get the latest mac address, User can use a LAA */ 800 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, 801 IXGBE_ETH_LENGTH_OF_ADDRESS); 802 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 803 hw->addr_ctrl.rar_used_count = 1; 804 805 /* Prepare transmit descriptors and buffers */ 806 if (ixv_setup_transmit_structures(adapter)) { 807 device_printf(dev,"Could not setup transmit structures\n"); 808 ixv_stop(adapter); 809 return; 810 } 811 812 ixgbe_reset_hw(hw); 813 ixv_initialize_transmit_units(adapter); 814 815 /* Setup Multicast table */ 816 ixv_set_multi(adapter); 817 818 /* 819 ** Determine the correct mbuf pool 820 ** for doing jumbo/headersplit 821 */ 822 if (ifp->if_mtu > ETHERMTU) 823 adapter->rx_mbuf_sz = MJUMPAGESIZE; 824 else 825 adapter->rx_mbuf_sz = MCLBYTES; 826 827 /* Prepare receive descriptors and buffers */ 828 if (ixv_setup_receive_structures(adapter)) { 829 device_printf(dev,"Could not setup receive structures\n"); 830 ixv_stop(adapter); 831 return; 832 } 833 834 /* Configure RX settings */ 835 ixv_initialize_receive_units(adapter); 836 837 /* Enable Enhanced MSIX mode */ 838 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 839 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 840 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 841 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 842 843 /* Set the various hardware offload abilities */ 844 ifp->if_hwassist = 0; 845 if (ifp->if_capenable & IFCAP_TSO4) 846 ifp->if_hwassist |= CSUM_TSO; 847 if (ifp->if_capenable & IFCAP_TXCSUM) { 848 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 849 #if __FreeBSD_version >= 800000 850 ifp->if_hwassist |= CSUM_SCTP; 851 #endif 852 } 853 854 /* Set MTU size */ 855 if (ifp->if_mtu > ETHERMTU) { 856 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 857 mhadd &= ~IXGBE_MHADD_MFS_MASK; 858 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 859 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 860 } 861 862 /* Set up VLAN offload and filter */ 863 ixv_setup_vlan_support(adapter); 864 865 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 866 867 /* Set up MSI/X routing */ 868 ixv_configure_ivars(adapter); 869 870 /* Set up auto-mask */ 871 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 872 873 /* Set moderation on the Link interrupt */ 874 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 875 876 /* Stats init */ 877 ixv_init_stats(adapter); 878 879 /* Config/Enable Link */ 880 ixv_config_link(adapter); 881 882 /* And now turn on interrupts */ 883 ixv_enable_intr(adapter); 884 885 /* Now inform the stack we're ready */ 886 ifp->if_drv_flags |= IFF_DRV_RUNNING; 887 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 888 889 return; 890 } 891 892 static void 893 ixv_init(void *arg) 894 { 895 struct adapter *adapter = arg; 896 897 IXV_CORE_LOCK(adapter); 898 ixv_init_locked(adapter); 899 IXV_CORE_UNLOCK(adapter); 900 return; 901 } 902 903 904 /* 905 ** 906 ** MSIX Interrupt Handlers and Tasklets 907 ** 908 */ 909 910 static inline void 911 ixv_enable_queue(struct adapter *adapter, u32 vector) 912 { 913 struct ixgbe_hw *hw = &adapter->hw; 914 u32 queue = 1 << vector; 915 u32 mask; 916 917 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 918 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 919 } 920 921 static inline void 922 ixv_disable_queue(struct adapter *adapter, u32 vector) 923 { 924 struct ixgbe_hw *hw = &adapter->hw; 925 u64 queue = (u64)(1 << vector); 926 u32 mask; 927 928 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 929 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 930 } 931 932 static inline void 933 ixv_rearm_queues(struct adapter *adapter, u64 queues) 934 { 935 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 936 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 937 } 938 939 940 static void 941 ixv_handle_que(void *context) 942 { 943 struct ix_queue *que = context; 944 struct adapter *adapter = que->adapter; 945 struct tx_ring *txr = que->txr; 946 struct ifnet *ifp = adapter->ifp; 947 bool more; 948 949 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 950 more = ixv_rxeof(que, adapter->rx_process_limit); 951 IXV_TX_LOCK(txr); 952 ixv_txeof(txr); 953 #if __FreeBSD_version >= 800000 954 if (!drbr_empty(ifp, txr->br)) 955 ixv_mq_start_locked(ifp, txr, NULL); 956 #else 957 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 958 ixv_start_locked(txr, ifp); 959 #endif 960 IXV_TX_UNLOCK(txr); 961 if (more) { 962 taskqueue_enqueue(que->tq, &que->que_task); 963 return; 964 } 965 } 966 967 /* Reenable this interrupt */ 968 ixv_enable_queue(adapter, que->msix); 969 return; 970 } 971 972 /********************************************************************* 973 * 974 * MSI Queue Interrupt Service routine 975 * 976 **********************************************************************/ 977 void 978 ixv_msix_que(void *arg) 979 { 980 struct ix_queue *que = arg; 981 struct adapter *adapter = que->adapter; 982 struct tx_ring *txr = que->txr; 983 struct rx_ring *rxr = que->rxr; 984 bool more_tx, more_rx; 985 u32 newitr = 0; 986 987 ixv_disable_queue(adapter, que->msix); 988 ++que->irqs; 989 990 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 991 992 IXV_TX_LOCK(txr); 993 more_tx = ixv_txeof(txr); 994 IXV_TX_UNLOCK(txr); 995 996 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 997 998 /* Do AIM now? */ 999 1000 if (ixv_enable_aim == FALSE) 1001 goto no_calc; 1002 /* 1003 ** Do Adaptive Interrupt Moderation: 1004 ** - Write out last calculated setting 1005 ** - Calculate based on average size over 1006 ** the last interval. 1007 */ 1008 if (que->eitr_setting) 1009 IXGBE_WRITE_REG(&adapter->hw, 1010 IXGBE_VTEITR(que->msix), 1011 que->eitr_setting); 1012 1013 que->eitr_setting = 0; 1014 1015 /* Idle, do nothing */ 1016 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1017 goto no_calc; 1018 1019 if ((txr->bytes) && (txr->packets)) 1020 newitr = txr->bytes/txr->packets; 1021 if ((rxr->bytes) && (rxr->packets)) 1022 newitr = max(newitr, 1023 (rxr->bytes / rxr->packets)); 1024 newitr += 24; /* account for hardware frame, crc */ 1025 1026 /* set an upper boundary */ 1027 newitr = min(newitr, 3000); 1028 1029 /* Be nice to the mid range */ 1030 if ((newitr > 300) && (newitr < 1200)) 1031 newitr = (newitr / 3); 1032 else 1033 newitr = (newitr / 2); 1034 1035 newitr |= newitr << 16; 1036 1037 /* save for next interrupt */ 1038 que->eitr_setting = newitr; 1039 1040 /* Reset state */ 1041 txr->bytes = 0; 1042 txr->packets = 0; 1043 rxr->bytes = 0; 1044 rxr->packets = 0; 1045 1046 no_calc: 1047 if (more_tx || more_rx) 1048 taskqueue_enqueue(que->tq, &que->que_task); 1049 else /* Reenable this interrupt */ 1050 ixv_enable_queue(adapter, que->msix); 1051 return; 1052 } 1053 1054 static void 1055 ixv_msix_mbx(void *arg) 1056 { 1057 struct adapter *adapter = arg; 1058 struct ixgbe_hw *hw = &adapter->hw; 1059 u32 reg; 1060 1061 ++adapter->mbx_irq; 1062 1063 /* First get the cause */ 1064 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1065 /* Clear interrupt with write */ 1066 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1067 1068 /* Link status change */ 1069 if (reg & IXGBE_EICR_LSC) 1070 taskqueue_enqueue(adapter->tq, &adapter->mbx_task); 1071 1072 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1073 return; 1074 } 1075 1076 /********************************************************************* 1077 * 1078 * Media Ioctl callback 1079 * 1080 * This routine is called whenever the user queries the status of 1081 * the interface using ifconfig. 1082 * 1083 **********************************************************************/ 1084 static void 1085 ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1086 { 1087 struct adapter *adapter = ifp->if_softc; 1088 1089 INIT_DEBUGOUT("ixv_media_status: begin"); 1090 IXV_CORE_LOCK(adapter); 1091 ixv_update_link_status(adapter); 1092 1093 ifmr->ifm_status = IFM_AVALID; 1094 ifmr->ifm_active = IFM_ETHER; 1095 1096 if (!adapter->link_active) { 1097 IXV_CORE_UNLOCK(adapter); 1098 return; 1099 } 1100 1101 ifmr->ifm_status |= IFM_ACTIVE; 1102 1103 switch (adapter->link_speed) { 1104 case IXGBE_LINK_SPEED_1GB_FULL: 1105 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1106 break; 1107 case IXGBE_LINK_SPEED_10GB_FULL: 1108 ifmr->ifm_active |= IFM_FDX; 1109 break; 1110 } 1111 1112 IXV_CORE_UNLOCK(adapter); 1113 1114 return; 1115 } 1116 1117 /********************************************************************* 1118 * 1119 * Media Ioctl callback 1120 * 1121 * This routine is called when the user changes speed/duplex using 1122 * media/mediopt option with ifconfig. 1123 * 1124 **********************************************************************/ 1125 static int 1126 ixv_media_change(struct ifnet * ifp) 1127 { 1128 struct adapter *adapter = ifp->if_softc; 1129 struct ifmedia *ifm = &adapter->media; 1130 1131 INIT_DEBUGOUT("ixv_media_change: begin"); 1132 1133 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1134 return (EINVAL); 1135 1136 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1137 case IFM_AUTO: 1138 break; 1139 default: 1140 device_printf(adapter->dev, "Only auto media type\n"); 1141 return (EINVAL); 1142 } 1143 1144 return (0); 1145 } 1146 1147 /********************************************************************* 1148 * 1149 * This routine maps the mbufs to tx descriptors, allowing the 1150 * TX engine to transmit the packets. 1151 * - return 0 on success, positive on failure 1152 * 1153 **********************************************************************/ 1154 1155 static int 1156 ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1157 { 1158 struct adapter *adapter = txr->adapter; 1159 u32 olinfo_status = 0, cmd_type_len; 1160 u32 paylen = 0; 1161 int i, j, error, nsegs; 1162 int first, last = 0; 1163 struct mbuf *m_head; 1164 bus_dma_segment_t segs[32]; 1165 bus_dmamap_t map; 1166 struct ixv_tx_buf *txbuf, *txbuf_mapped; 1167 union ixgbe_adv_tx_desc *txd = NULL; 1168 1169 m_head = *m_headp; 1170 1171 /* Basic descriptor defines */ 1172 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1173 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1174 1175 if (m_head->m_flags & M_VLANTAG) 1176 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1177 1178 /* 1179 * Important to capture the first descriptor 1180 * used because it will contain the index of 1181 * the one we tell the hardware to report back 1182 */ 1183 first = txr->next_avail_desc; 1184 txbuf = &txr->tx_buffers[first]; 1185 txbuf_mapped = txbuf; 1186 map = txbuf->map; 1187 1188 /* 1189 * Map the packet for DMA. 1190 */ 1191 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1192 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1193 1194 if (error == EFBIG) { 1195 struct mbuf *m; 1196 1197 m = m_defrag(*m_headp, M_DONTWAIT); 1198 if (m == NULL) { 1199 adapter->mbuf_defrag_failed++; 1200 m_freem(*m_headp); 1201 *m_headp = NULL; 1202 return (ENOBUFS); 1203 } 1204 *m_headp = m; 1205 1206 /* Try it again */ 1207 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1208 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1209 1210 if (error == ENOMEM) { 1211 adapter->no_tx_dma_setup++; 1212 return (error); 1213 } else if (error != 0) { 1214 adapter->no_tx_dma_setup++; 1215 m_freem(*m_headp); 1216 *m_headp = NULL; 1217 return (error); 1218 } 1219 } else if (error == ENOMEM) { 1220 adapter->no_tx_dma_setup++; 1221 return (error); 1222 } else if (error != 0) { 1223 adapter->no_tx_dma_setup++; 1224 m_freem(*m_headp); 1225 *m_headp = NULL; 1226 return (error); 1227 } 1228 1229 /* Make certain there are enough descriptors */ 1230 if (nsegs > txr->tx_avail - 2) { 1231 txr->no_desc_avail++; 1232 error = ENOBUFS; 1233 goto xmit_fail; 1234 } 1235 m_head = *m_headp; 1236 1237 /* 1238 ** Set up the appropriate offload context 1239 ** this becomes the first descriptor of 1240 ** a packet. 1241 */ 1242 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1243 if (ixv_tso_setup(txr, m_head, &paylen)) { 1244 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1245 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1246 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1247 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1248 ++adapter->tso_tx; 1249 } else 1250 return (ENXIO); 1251 } else if (ixv_tx_ctx_setup(txr, m_head)) 1252 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1253 1254 /* Record payload length */ 1255 if (paylen == 0) 1256 olinfo_status |= m_head->m_pkthdr.len << 1257 IXGBE_ADVTXD_PAYLEN_SHIFT; 1258 1259 i = txr->next_avail_desc; 1260 for (j = 0; j < nsegs; j++) { 1261 bus_size_t seglen; 1262 bus_addr_t segaddr; 1263 1264 txbuf = &txr->tx_buffers[i]; 1265 txd = &txr->tx_base[i]; 1266 seglen = segs[j].ds_len; 1267 segaddr = htole64(segs[j].ds_addr); 1268 1269 txd->read.buffer_addr = segaddr; 1270 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1271 cmd_type_len |seglen); 1272 txd->read.olinfo_status = htole32(olinfo_status); 1273 last = i; /* descriptor that will get completion IRQ */ 1274 1275 if (++i == adapter->num_tx_desc) 1276 i = 0; 1277 1278 txbuf->m_head = NULL; 1279 txbuf->eop_index = -1; 1280 } 1281 1282 txd->read.cmd_type_len |= 1283 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1284 txr->tx_avail -= nsegs; 1285 txr->next_avail_desc = i; 1286 1287 txbuf->m_head = m_head; 1288 txbuf->map = map; 1289 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1290 1291 /* Set the index of the descriptor that will be marked done */ 1292 txbuf = &txr->tx_buffers[first]; 1293 txbuf->eop_index = last; 1294 1295 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1296 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1297 /* 1298 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1299 * hardware that this frame is available to transmit. 1300 */ 1301 ++txr->total_packets; 1302 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1303 1304 return (0); 1305 1306 xmit_fail: 1307 bus_dmamap_unload(txr->txtag, txbuf->map); 1308 return (error); 1309 1310 } 1311 1312 1313 /********************************************************************* 1314 * Multicast Update 1315 * 1316 * This routine is called whenever multicast address list is updated. 1317 * 1318 **********************************************************************/ 1319 #define IXGBE_RAR_ENTRIES 16 1320 1321 static void 1322 ixv_set_multi(struct adapter *adapter) 1323 { 1324 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1325 u8 *update_ptr; 1326 struct ifmultiaddr *ifma; 1327 int mcnt = 0; 1328 struct ifnet *ifp = adapter->ifp; 1329 1330 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1331 1332 #if __FreeBSD_version < 800000 1333 IF_ADDR_LOCK(ifp); 1334 #else 1335 if_maddr_rlock(ifp); 1336 #endif 1337 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1338 if (ifma->ifma_addr->sa_family != AF_LINK) 1339 continue; 1340 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1341 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1342 IXGBE_ETH_LENGTH_OF_ADDRESS); 1343 mcnt++; 1344 } 1345 #if __FreeBSD_version < 800000 1346 IF_ADDR_UNLOCK(ifp); 1347 #else 1348 if_maddr_runlock(ifp); 1349 #endif 1350 1351 update_ptr = mta; 1352 1353 ixgbe_update_mc_addr_list(&adapter->hw, 1354 update_ptr, mcnt, ixv_mc_array_itr); 1355 1356 return; 1357 } 1358 1359 /* 1360 * This is an iterator function now needed by the multicast 1361 * shared code. It simply feeds the shared code routine the 1362 * addresses in the array of ixv_set_multi() one by one. 1363 */ 1364 static u8 * 1365 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1366 { 1367 u8 *addr = *update_ptr; 1368 u8 *newptr; 1369 *vmdq = 0; 1370 1371 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1372 *update_ptr = newptr; 1373 return addr; 1374 } 1375 1376 /********************************************************************* 1377 * Timer routine 1378 * 1379 * This routine checks for link status,updates statistics, 1380 * and runs the watchdog check. 1381 * 1382 **********************************************************************/ 1383 1384 static void 1385 ixv_local_timer(void *arg) 1386 { 1387 struct adapter *adapter = arg; 1388 device_t dev = adapter->dev; 1389 struct tx_ring *txr = adapter->tx_rings; 1390 int i; 1391 1392 mtx_assert(&adapter->core_mtx, MA_OWNED); 1393 1394 ixv_update_link_status(adapter); 1395 1396 /* Stats Update */ 1397 ixv_update_stats(adapter); 1398 1399 /* 1400 * If the interface has been paused 1401 * then don't do the watchdog check 1402 */ 1403 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1404 goto out; 1405 /* 1406 ** Check for time since any descriptor was cleaned 1407 */ 1408 for (i = 0; i < adapter->num_queues; i++, txr++) { 1409 IXV_TX_LOCK(txr); 1410 if (txr->watchdog_check == FALSE) { 1411 IXV_TX_UNLOCK(txr); 1412 continue; 1413 } 1414 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG) 1415 goto hung; 1416 IXV_TX_UNLOCK(txr); 1417 } 1418 out: 1419 ixv_rearm_queues(adapter, adapter->que_mask); 1420 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1421 return; 1422 1423 hung: 1424 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1425 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1426 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1427 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1428 device_printf(dev,"TX(%d) desc avail = %d," 1429 "Next TX to Clean = %d\n", 1430 txr->me, txr->tx_avail, txr->next_to_clean); 1431 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1432 adapter->watchdog_events++; 1433 IXV_TX_UNLOCK(txr); 1434 ixv_init_locked(adapter); 1435 } 1436 1437 /* 1438 ** Note: this routine updates the OS on the link state 1439 ** the real check of the hardware only happens with 1440 ** a link interrupt. 1441 */ 1442 static void 1443 ixv_update_link_status(struct adapter *adapter) 1444 { 1445 struct ifnet *ifp = adapter->ifp; 1446 struct tx_ring *txr = adapter->tx_rings; 1447 device_t dev = adapter->dev; 1448 1449 1450 if (adapter->link_up){ 1451 if (adapter->link_active == FALSE) { 1452 if (bootverbose) 1453 device_printf(dev,"Link is up %d Gbps %s \n", 1454 ((adapter->link_speed == 128)? 10:1), 1455 "Full Duplex"); 1456 adapter->link_active = TRUE; 1457 if_link_state_change(ifp, LINK_STATE_UP); 1458 } 1459 } else { /* Link down */ 1460 if (adapter->link_active == TRUE) { 1461 if (bootverbose) 1462 device_printf(dev,"Link is Down\n"); 1463 if_link_state_change(ifp, LINK_STATE_DOWN); 1464 adapter->link_active = FALSE; 1465 for (int i = 0; i < adapter->num_queues; 1466 i++, txr++) 1467 txr->watchdog_check = FALSE; 1468 } 1469 } 1470 1471 return; 1472 } 1473 1474 1475 /********************************************************************* 1476 * 1477 * This routine disables all traffic on the adapter by issuing a 1478 * global reset on the MAC and deallocates TX/RX buffers. 1479 * 1480 **********************************************************************/ 1481 1482 static void 1483 ixv_stop(void *arg) 1484 { 1485 struct ifnet *ifp; 1486 struct adapter *adapter = arg; 1487 struct ixgbe_hw *hw = &adapter->hw; 1488 ifp = adapter->ifp; 1489 1490 mtx_assert(&adapter->core_mtx, MA_OWNED); 1491 1492 INIT_DEBUGOUT("ixv_stop: begin\n"); 1493 ixv_disable_intr(adapter); 1494 1495 /* Tell the stack that the interface is no longer active */ 1496 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1497 1498 ixgbe_reset_hw(hw); 1499 adapter->hw.adapter_stopped = FALSE; 1500 ixgbe_stop_adapter(hw); 1501 callout_stop(&adapter->timer); 1502 1503 /* reprogram the RAR[0] in case user changed it. */ 1504 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1505 1506 return; 1507 } 1508 1509 1510 /********************************************************************* 1511 * 1512 * Determine hardware revision. 1513 * 1514 **********************************************************************/ 1515 static void 1516 ixv_identify_hardware(struct adapter *adapter) 1517 { 1518 device_t dev = adapter->dev; 1519 u16 pci_cmd_word; 1520 1521 /* 1522 ** Make sure BUSMASTER is set, on a VM under 1523 ** KVM it may not be and will break things. 1524 */ 1525 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1526 if (!((pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 1527 (pci_cmd_word & PCIM_CMD_MEMEN))) { 1528 INIT_DEBUGOUT("Memory Access and/or Bus Master " 1529 "bits were not set!\n"); 1530 pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 1531 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); 1532 } 1533 1534 /* Save off the information about this board */ 1535 adapter->hw.vendor_id = pci_get_vendor(dev); 1536 adapter->hw.device_id = pci_get_device(dev); 1537 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1538 adapter->hw.subsystem_vendor_id = 1539 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1540 adapter->hw.subsystem_device_id = 1541 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1542 1543 return; 1544 } 1545 1546 /********************************************************************* 1547 * 1548 * Setup MSIX Interrupt resources and handlers 1549 * 1550 **********************************************************************/ 1551 static int 1552 ixv_allocate_msix(struct adapter *adapter) 1553 { 1554 device_t dev = adapter->dev; 1555 struct ix_queue *que = adapter->queues; 1556 int error, rid, vector = 0; 1557 1558 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1559 rid = vector + 1; 1560 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1561 RF_SHAREABLE | RF_ACTIVE); 1562 if (que->res == NULL) { 1563 device_printf(dev,"Unable to allocate" 1564 " bus resource: que interrupt [%d]\n", vector); 1565 return (ENXIO); 1566 } 1567 /* Set the handler function */ 1568 error = bus_setup_intr(dev, que->res, 1569 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1570 ixv_msix_que, que, &que->tag); 1571 if (error) { 1572 que->res = NULL; 1573 device_printf(dev, "Failed to register QUE handler"); 1574 return (error); 1575 } 1576 #if __FreeBSD_version >= 800504 1577 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1578 #endif 1579 que->msix = vector; 1580 adapter->que_mask |= (u64)(1 << que->msix); 1581 /* 1582 ** Bind the msix vector, and thus the 1583 ** ring to the corresponding cpu. 1584 */ 1585 if (adapter->num_queues > 1) 1586 bus_bind_intr(dev, que->res, i); 1587 1588 ixgbe_task_init(&que->que_task, ixv_handle_que, que); 1589 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT, 1590 taskqueue_thread_enqueue, &que->tq); 1591 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1592 device_get_nameunit(adapter->dev)); 1593 } 1594 1595 /* and Mailbox */ 1596 rid = vector + 1; 1597 adapter->res = bus_alloc_resource_any(dev, 1598 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1599 if (!adapter->res) { 1600 device_printf(dev,"Unable to allocate" 1601 " bus resource: MBX interrupt [%d]\n", rid); 1602 return (ENXIO); 1603 } 1604 /* Set the mbx handler function */ 1605 error = bus_setup_intr(dev, adapter->res, 1606 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1607 ixv_msix_mbx, adapter, &adapter->tag); 1608 if (error) { 1609 adapter->res = NULL; 1610 device_printf(dev, "Failed to register LINK handler"); 1611 return (error); 1612 } 1613 #if __FreeBSD_version >= 800504 1614 bus_describe_intr(dev, adapter->res, adapter->tag, "mbx"); 1615 #endif 1616 adapter->mbxvec = vector; 1617 /* Tasklets for Mailbox */ 1618 ixgbe_task_init(&adapter->mbx_task, ixv_handle_mbx, adapter); 1619 adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT, 1620 taskqueue_thread_enqueue, &adapter->tq); 1621 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq", 1622 device_get_nameunit(adapter->dev)); 1623 /* 1624 ** Due to a broken design QEMU will fail to properly 1625 ** enable the guest for MSIX unless the vectors in 1626 ** the table are all set up, so we must rewrite the 1627 ** ENABLE in the MSIX control register again at this 1628 ** point to cause it to successfully initialize us. 1629 */ 1630 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1631 int msix_ctrl; 1632 pci_find_cap(dev, PCIY_MSIX, &rid); 1633 rid += PCIR_MSIX_CTRL; 1634 msix_ctrl = pci_read_config(dev, rid, 2); 1635 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1636 pci_write_config(dev, rid, msix_ctrl, 2); 1637 } 1638 1639 return (0); 1640 } 1641 1642 /* 1643 * Setup MSIX resources, note that the VF 1644 * device MUST use MSIX, there is no fallback. 1645 */ 1646 static int 1647 ixv_setup_msix(struct adapter *adapter) 1648 { 1649 device_t dev = adapter->dev; 1650 int rid, vectors, want = 2; 1651 1652 1653 /* First try MSI/X */ 1654 rid = PCIR_BAR(3); 1655 adapter->msix_mem = bus_alloc_resource_any(dev, 1656 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1657 if (!adapter->msix_mem) { 1658 device_printf(adapter->dev, 1659 "Unable to map MSIX table \n"); 1660 goto out; 1661 } 1662 1663 vectors = pci_msix_count(dev); 1664 if (vectors < 2) { 1665 bus_release_resource(dev, SYS_RES_MEMORY, 1666 rid, adapter->msix_mem); 1667 adapter->msix_mem = NULL; 1668 goto out; 1669 } 1670 1671 /* 1672 ** Want two vectors: one for a queue, 1673 ** plus an additional for mailbox. 1674 */ 1675 if (pci_alloc_msix(dev, &want) == 0) { 1676 device_printf(adapter->dev, 1677 "Using MSIX interrupts with %d vectors\n", want); 1678 return (want); 1679 } 1680 out: 1681 device_printf(adapter->dev,"MSIX config error\n"); 1682 return (ENXIO); 1683 } 1684 1685 1686 static int 1687 ixv_allocate_pci_resources(struct adapter *adapter) 1688 { 1689 int rid; 1690 device_t dev = adapter->dev; 1691 1692 rid = PCIR_BAR(0); 1693 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1694 &rid, RF_ACTIVE); 1695 1696 if (!(adapter->pci_mem)) { 1697 device_printf(dev,"Unable to allocate bus resource: memory\n"); 1698 return (ENXIO); 1699 } 1700 1701 adapter->osdep.mem_bus_space_tag = 1702 rman_get_bustag(adapter->pci_mem); 1703 adapter->osdep.mem_bus_space_handle = 1704 rman_get_bushandle(adapter->pci_mem); 1705 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 1706 1707 adapter->num_queues = 1; 1708 adapter->hw.back = &adapter->osdep; 1709 1710 /* 1711 ** Now setup MSI/X, should 1712 ** return us the number of 1713 ** configured vectors. 1714 */ 1715 adapter->msix = ixv_setup_msix(adapter); 1716 if (adapter->msix == ENXIO) 1717 return (ENXIO); 1718 else 1719 return (0); 1720 } 1721 1722 static void 1723 ixv_free_pci_resources(struct adapter * adapter) 1724 { 1725 struct ix_queue *que = adapter->queues; 1726 device_t dev = adapter->dev; 1727 int rid, memrid; 1728 1729 memrid = PCIR_BAR(MSIX_BAR); 1730 1731 /* 1732 ** There is a slight possibility of a failure mode 1733 ** in attach that will result in entering this function 1734 ** before interrupt resources have been initialized, and 1735 ** in that case we do not want to execute the loops below 1736 ** We can detect this reliably by the state of the adapter 1737 ** res pointer. 1738 */ 1739 if (adapter->res == NULL) 1740 goto mem; 1741 1742 /* 1743 ** Release all msix queue resources: 1744 */ 1745 for (int i = 0; i < adapter->num_queues; i++, que++) { 1746 rid = que->msix + 1; 1747 if (que->tag != NULL) { 1748 bus_teardown_intr(dev, que->res, que->tag); 1749 que->tag = NULL; 1750 } 1751 if (que->res != NULL) 1752 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 1753 } 1754 1755 1756 /* Clean the Legacy or Link interrupt last */ 1757 if (adapter->mbxvec) /* we are doing MSIX */ 1758 rid = adapter->mbxvec + 1; 1759 else 1760 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1761 1762 if (adapter->tag != NULL) { 1763 bus_teardown_intr(dev, adapter->res, adapter->tag); 1764 adapter->tag = NULL; 1765 } 1766 if (adapter->res != NULL) 1767 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 1768 1769 mem: 1770 if (adapter->msix) 1771 pci_release_msi(dev); 1772 1773 if (adapter->msix_mem != NULL) 1774 bus_release_resource(dev, SYS_RES_MEMORY, 1775 memrid, adapter->msix_mem); 1776 1777 if (adapter->pci_mem != NULL) 1778 bus_release_resource(dev, SYS_RES_MEMORY, 1779 PCIR_BAR(0), adapter->pci_mem); 1780 1781 return; 1782 } 1783 1784 /********************************************************************* 1785 * 1786 * Setup networking device structure and register an interface. 1787 * 1788 **********************************************************************/ 1789 static void 1790 ixv_setup_interface(device_t dev, struct adapter *adapter) 1791 { 1792 struct ifnet *ifp; 1793 1794 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1795 1796 ifp = adapter->ifp = if_alloc(IFT_ETHER); 1797 if (ifp == NULL) 1798 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 1799 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1800 ifp->if_mtu = ETHERMTU; 1801 ifp->if_baudrate = 1000000000; 1802 ifp->if_init = ixv_init; 1803 ifp->if_softc = adapter; 1804 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1805 ifp->if_ioctl = ixv_ioctl; 1806 #if __FreeBSD_version >= 800000 1807 ifp->if_transmit = ixv_mq_start; 1808 ifp->if_qflush = ixv_qflush; 1809 #else 1810 ifp->if_start = ixv_start; 1811 #endif 1812 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1813 1814 ether_ifattach(ifp, adapter->hw.mac.addr); 1815 1816 adapter->max_frame_size = 1817 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1818 1819 /* 1820 * Tell the upper layer(s) we support long frames. 1821 */ 1822 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1823 1824 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM; 1825 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 1826 ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_LRO; 1827 1828 ifp->if_capenable = ifp->if_capabilities; 1829 1830 /* 1831 * Specify the media types supported by this adapter and register 1832 * callbacks to update media and link information 1833 */ 1834 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1835 ixv_media_status); 1836 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1837 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1838 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1839 1840 return; 1841 } 1842 1843 static void 1844 ixv_config_link(struct adapter *adapter) 1845 { 1846 struct ixgbe_hw *hw = &adapter->hw; 1847 u32 autoneg, err = 0; 1848 bool negotiate = TRUE; 1849 1850 if (hw->mac.ops.check_link) 1851 err = hw->mac.ops.check_link(hw, &autoneg, 1852 &adapter->link_up, FALSE); 1853 if (err) 1854 goto out; 1855 1856 if (hw->mac.ops.setup_link) 1857 err = hw->mac.ops.setup_link(hw, autoneg, 1858 negotiate, adapter->link_up); 1859 out: 1860 return; 1861 } 1862 1863 /******************************************************************** 1864 * Manage DMA'able memory. 1865 *******************************************************************/ 1866 static void 1867 ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 1868 { 1869 if (error) 1870 return; 1871 *(bus_addr_t *) arg = segs->ds_addr; 1872 return; 1873 } 1874 1875 static int 1876 ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 1877 struct ixv_dma_alloc *dma, int mapflags) 1878 { 1879 device_t dev = adapter->dev; 1880 int r; 1881 1882 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 1883 DBA_ALIGN, 0, /* alignment, bounds */ 1884 BUS_SPACE_MAXADDR, /* lowaddr */ 1885 BUS_SPACE_MAXADDR, /* highaddr */ 1886 NULL, NULL, /* filter, filterarg */ 1887 size, /* maxsize */ 1888 1, /* nsegments */ 1889 size, /* maxsegsize */ 1890 BUS_DMA_ALLOCNOW, /* flags */ 1891 NULL, /* lockfunc */ 1892 NULL, /* lockfuncarg */ 1893 &dma->dma_tag); 1894 if (r != 0) { 1895 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; " 1896 "error %u\n", r); 1897 goto fail_0; 1898 } 1899 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 1900 BUS_DMA_NOWAIT, &dma->dma_map); 1901 if (r != 0) { 1902 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; " 1903 "error %u\n", r); 1904 goto fail_1; 1905 } 1906 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1907 size, 1908 ixv_dmamap_cb, 1909 &dma->dma_paddr, 1910 mapflags | BUS_DMA_NOWAIT); 1911 if (r != 0) { 1912 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; " 1913 "error %u\n", r); 1914 goto fail_2; 1915 } 1916 dma->dma_size = size; 1917 return (0); 1918 fail_2: 1919 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1920 fail_1: 1921 bus_dma_tag_destroy(dma->dma_tag); 1922 fail_0: 1923 dma->dma_map = NULL; 1924 dma->dma_tag = NULL; 1925 return (r); 1926 } 1927 1928 static void 1929 ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 1930 { 1931 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 1932 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1933 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1934 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1935 bus_dma_tag_destroy(dma->dma_tag); 1936 } 1937 1938 1939 /********************************************************************* 1940 * 1941 * Allocate memory for the transmit and receive rings, and then 1942 * the descriptors associated with each, called only once at attach. 1943 * 1944 **********************************************************************/ 1945 static int 1946 ixv_allocate_queues(struct adapter *adapter) 1947 { 1948 device_t dev = adapter->dev; 1949 struct ix_queue *que; 1950 struct tx_ring *txr; 1951 struct rx_ring *rxr; 1952 int rsize, tsize, error = 0; 1953 int txconf = 0, rxconf = 0; 1954 1955 /* First allocate the top level queue structs */ 1956 if (!(adapter->queues = 1957 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 1958 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1959 device_printf(dev, "Unable to allocate queue memory\n"); 1960 error = ENOMEM; 1961 goto fail; 1962 } 1963 1964 /* First allocate the TX ring struct memory */ 1965 if (!(adapter->tx_rings = 1966 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 1967 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1968 device_printf(dev, "Unable to allocate TX ring memory\n"); 1969 error = ENOMEM; 1970 goto tx_fail; 1971 } 1972 1973 /* Next allocate the RX */ 1974 if (!(adapter->rx_rings = 1975 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 1976 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1977 device_printf(dev, "Unable to allocate RX ring memory\n"); 1978 error = ENOMEM; 1979 goto rx_fail; 1980 } 1981 1982 /* For the ring itself */ 1983 tsize = roundup2(adapter->num_tx_desc * 1984 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 1985 1986 /* 1987 * Now set up the TX queues, txconf is needed to handle the 1988 * possibility that things fail midcourse and we need to 1989 * undo memory gracefully 1990 */ 1991 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 1992 /* Set up some basics */ 1993 txr = &adapter->tx_rings[i]; 1994 txr->adapter = adapter; 1995 txr->me = i; 1996 1997 /* Initialize the TX side lock */ 1998 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 1999 device_get_nameunit(dev), txr->me); 2000 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2001 2002 if (ixv_dma_malloc(adapter, tsize, 2003 &txr->txdma, BUS_DMA_NOWAIT)) { 2004 device_printf(dev, 2005 "Unable to allocate TX Descriptor memory\n"); 2006 error = ENOMEM; 2007 goto err_tx_desc; 2008 } 2009 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2010 bzero((void *)txr->tx_base, tsize); 2011 2012 /* Now allocate transmit buffers for the ring */ 2013 if (ixv_allocate_transmit_buffers(txr)) { 2014 device_printf(dev, 2015 "Critical Failure setting up transmit buffers\n"); 2016 error = ENOMEM; 2017 goto err_tx_desc; 2018 } 2019 #if __FreeBSD_version >= 800000 2020 /* Allocate a buf ring */ 2021 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2022 M_WAITOK, &txr->tx_mtx); 2023 if (txr->br == NULL) { 2024 device_printf(dev, 2025 "Critical Failure setting up buf ring\n"); 2026 error = ENOMEM; 2027 goto err_tx_desc; 2028 } 2029 #endif 2030 } 2031 2032 /* 2033 * Next the RX queues... 2034 */ 2035 rsize = roundup2(adapter->num_rx_desc * 2036 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2037 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2038 rxr = &adapter->rx_rings[i]; 2039 /* Set up some basics */ 2040 rxr->adapter = adapter; 2041 rxr->me = i; 2042 2043 /* Initialize the RX side lock */ 2044 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2045 device_get_nameunit(dev), rxr->me); 2046 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2047 2048 if (ixv_dma_malloc(adapter, rsize, 2049 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2050 device_printf(dev, 2051 "Unable to allocate RxDescriptor memory\n"); 2052 error = ENOMEM; 2053 goto err_rx_desc; 2054 } 2055 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2056 bzero((void *)rxr->rx_base, rsize); 2057 2058 /* Allocate receive buffers for the ring*/ 2059 if (ixv_allocate_receive_buffers(rxr)) { 2060 device_printf(dev, 2061 "Critical Failure setting up receive buffers\n"); 2062 error = ENOMEM; 2063 goto err_rx_desc; 2064 } 2065 } 2066 2067 /* 2068 ** Finally set up the queue holding structs 2069 */ 2070 for (int i = 0; i < adapter->num_queues; i++) { 2071 que = &adapter->queues[i]; 2072 que->adapter = adapter; 2073 que->txr = &adapter->tx_rings[i]; 2074 que->rxr = &adapter->rx_rings[i]; 2075 } 2076 2077 return (0); 2078 2079 err_rx_desc: 2080 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2081 ixv_dma_free(adapter, &rxr->rxdma); 2082 err_tx_desc: 2083 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2084 ixv_dma_free(adapter, &txr->txdma); 2085 free(adapter->rx_rings, M_DEVBUF); 2086 rx_fail: 2087 free(adapter->tx_rings, M_DEVBUF); 2088 tx_fail: 2089 free(adapter->queues, M_DEVBUF); 2090 fail: 2091 return (error); 2092 } 2093 2094 2095 /********************************************************************* 2096 * 2097 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2098 * the information needed to transmit a packet on the wire. This is 2099 * called only once at attach, setup is done every reset. 2100 * 2101 **********************************************************************/ 2102 static int 2103 ixv_allocate_transmit_buffers(struct tx_ring *txr) 2104 { 2105 struct adapter *adapter = txr->adapter; 2106 device_t dev = adapter->dev; 2107 struct ixv_tx_buf *txbuf; 2108 int error, i; 2109 2110 /* 2111 * Setup DMA descriptor areas. 2112 */ 2113 if ((error = bus_dma_tag_create(NULL, /* parent */ 2114 1, 0, /* alignment, bounds */ 2115 BUS_SPACE_MAXADDR, /* lowaddr */ 2116 BUS_SPACE_MAXADDR, /* highaddr */ 2117 NULL, NULL, /* filter, filterarg */ 2118 IXV_TSO_SIZE, /* maxsize */ 2119 32, /* nsegments */ 2120 PAGE_SIZE, /* maxsegsize */ 2121 0, /* flags */ 2122 NULL, /* lockfunc */ 2123 NULL, /* lockfuncarg */ 2124 &txr->txtag))) { 2125 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2126 goto fail; 2127 } 2128 2129 if (!(txr->tx_buffers = 2130 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2131 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2132 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2133 error = ENOMEM; 2134 goto fail; 2135 } 2136 2137 /* Create the descriptor buffer dma maps */ 2138 txbuf = txr->tx_buffers; 2139 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2140 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2141 if (error != 0) { 2142 device_printf(dev, "Unable to create TX DMA map\n"); 2143 goto fail; 2144 } 2145 } 2146 2147 return 0; 2148 fail: 2149 /* We free all, it handles case where we are in the middle */ 2150 ixv_free_transmit_structures(adapter); 2151 return (error); 2152 } 2153 2154 /********************************************************************* 2155 * 2156 * Initialize a transmit ring. 2157 * 2158 **********************************************************************/ 2159 static void 2160 ixv_setup_transmit_ring(struct tx_ring *txr) 2161 { 2162 struct adapter *adapter = txr->adapter; 2163 struct ixv_tx_buf *txbuf; 2164 int i; 2165 2166 /* Clear the old ring contents */ 2167 IXV_TX_LOCK(txr); 2168 bzero((void *)txr->tx_base, 2169 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2170 /* Reset indices */ 2171 txr->next_avail_desc = 0; 2172 txr->next_to_clean = 0; 2173 2174 /* Free any existing tx buffers. */ 2175 txbuf = txr->tx_buffers; 2176 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2177 if (txbuf->m_head != NULL) { 2178 bus_dmamap_sync(txr->txtag, txbuf->map, 2179 BUS_DMASYNC_POSTWRITE); 2180 bus_dmamap_unload(txr->txtag, txbuf->map); 2181 m_freem(txbuf->m_head); 2182 txbuf->m_head = NULL; 2183 } 2184 /* Clear the EOP index */ 2185 txbuf->eop_index = -1; 2186 } 2187 2188 /* Set number of descriptors available */ 2189 txr->tx_avail = adapter->num_tx_desc; 2190 2191 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2192 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2193 IXV_TX_UNLOCK(txr); 2194 } 2195 2196 /********************************************************************* 2197 * 2198 * Initialize all transmit rings. 2199 * 2200 **********************************************************************/ 2201 static int 2202 ixv_setup_transmit_structures(struct adapter *adapter) 2203 { 2204 struct tx_ring *txr = adapter->tx_rings; 2205 2206 for (int i = 0; i < adapter->num_queues; i++, txr++) 2207 ixv_setup_transmit_ring(txr); 2208 2209 return (0); 2210 } 2211 2212 /********************************************************************* 2213 * 2214 * Enable transmit unit. 2215 * 2216 **********************************************************************/ 2217 static void 2218 ixv_initialize_transmit_units(struct adapter *adapter) 2219 { 2220 struct tx_ring *txr = adapter->tx_rings; 2221 struct ixgbe_hw *hw = &adapter->hw; 2222 2223 2224 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2225 u64 tdba = txr->txdma.dma_paddr; 2226 u32 txctrl, txdctl; 2227 2228 /* Set WTHRESH to 8, burst writeback */ 2229 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2230 txdctl |= (8 << 16); 2231 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2232 /* Now enable */ 2233 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2234 txdctl |= IXGBE_TXDCTL_ENABLE; 2235 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2236 2237 /* Set the HW Tx Head and Tail indices */ 2238 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2239 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2240 2241 /* Setup Transmit Descriptor Cmd Settings */ 2242 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2243 txr->watchdog_check = FALSE; 2244 2245 /* Set Ring parameters */ 2246 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2247 (tdba & 0x00000000ffffffffULL)); 2248 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2249 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2250 adapter->num_tx_desc * 2251 sizeof(struct ixgbe_legacy_tx_desc)); 2252 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2253 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 2254 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2255 break; 2256 } 2257 2258 return; 2259 } 2260 2261 /********************************************************************* 2262 * 2263 * Free all transmit rings. 2264 * 2265 **********************************************************************/ 2266 static void 2267 ixv_free_transmit_structures(struct adapter *adapter) 2268 { 2269 struct tx_ring *txr = adapter->tx_rings; 2270 2271 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2272 IXV_TX_LOCK(txr); 2273 ixv_free_transmit_buffers(txr); 2274 ixv_dma_free(adapter, &txr->txdma); 2275 IXV_TX_UNLOCK(txr); 2276 IXV_TX_LOCK_DESTROY(txr); 2277 } 2278 free(adapter->tx_rings, M_DEVBUF); 2279 } 2280 2281 /********************************************************************* 2282 * 2283 * Free transmit ring related data structures. 2284 * 2285 **********************************************************************/ 2286 static void 2287 ixv_free_transmit_buffers(struct tx_ring *txr) 2288 { 2289 struct adapter *adapter = txr->adapter; 2290 struct ixv_tx_buf *tx_buffer; 2291 int i; 2292 2293 INIT_DEBUGOUT("free_transmit_ring: begin"); 2294 2295 if (txr->tx_buffers == NULL) 2296 return; 2297 2298 tx_buffer = txr->tx_buffers; 2299 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2300 if (tx_buffer->m_head != NULL) { 2301 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2302 BUS_DMASYNC_POSTWRITE); 2303 bus_dmamap_unload(txr->txtag, 2304 tx_buffer->map); 2305 m_freem(tx_buffer->m_head); 2306 tx_buffer->m_head = NULL; 2307 if (tx_buffer->map != NULL) { 2308 bus_dmamap_destroy(txr->txtag, 2309 tx_buffer->map); 2310 tx_buffer->map = NULL; 2311 } 2312 } else if (tx_buffer->map != NULL) { 2313 bus_dmamap_unload(txr->txtag, 2314 tx_buffer->map); 2315 bus_dmamap_destroy(txr->txtag, 2316 tx_buffer->map); 2317 tx_buffer->map = NULL; 2318 } 2319 } 2320 #if __FreeBSD_version >= 800000 2321 if (txr->br != NULL) 2322 buf_ring_free(txr->br, M_DEVBUF); 2323 #endif 2324 if (txr->tx_buffers != NULL) { 2325 free(txr->tx_buffers, M_DEVBUF); 2326 txr->tx_buffers = NULL; 2327 } 2328 if (txr->txtag != NULL) { 2329 bus_dma_tag_destroy(txr->txtag); 2330 txr->txtag = NULL; 2331 } 2332 return; 2333 } 2334 2335 /********************************************************************* 2336 * 2337 * Advanced Context Descriptor setup for VLAN or CSUM 2338 * 2339 **********************************************************************/ 2340 2341 static boolean_t 2342 ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2343 { 2344 struct adapter *adapter = txr->adapter; 2345 struct ixgbe_adv_tx_context_desc *TXD; 2346 struct ixv_tx_buf *tx_buffer; 2347 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2348 struct ether_vlan_header *eh; 2349 struct ip *ip; 2350 struct ip6_hdr *ip6; 2351 int ehdrlen, ip_hlen = 0; 2352 u16 etype; 2353 u8 ipproto = 0; 2354 bool offload = TRUE; 2355 int ctxd = txr->next_avail_desc; 2356 u16 vtag = 0; 2357 2358 2359 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2360 offload = FALSE; 2361 2362 2363 tx_buffer = &txr->tx_buffers[ctxd]; 2364 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2365 2366 /* 2367 ** In advanced descriptors the vlan tag must 2368 ** be placed into the descriptor itself. 2369 */ 2370 if (mp->m_flags & M_VLANTAG) { 2371 vtag = htole16(mp->m_pkthdr.ether_vtag); 2372 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2373 } else if (offload == FALSE) 2374 return FALSE; 2375 2376 /* 2377 * Determine where frame payload starts. 2378 * Jump over vlan headers if already present, 2379 * helpful for QinQ too. 2380 */ 2381 eh = mtod(mp, struct ether_vlan_header *); 2382 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2383 etype = ntohs(eh->evl_proto); 2384 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2385 } else { 2386 etype = ntohs(eh->evl_encap_proto); 2387 ehdrlen = ETHER_HDR_LEN; 2388 } 2389 2390 /* Set the ether header length */ 2391 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2392 2393 switch (etype) { 2394 case ETHERTYPE_IP: 2395 ip = (struct ip *)(mp->m_data + ehdrlen); 2396 ip_hlen = ip->ip_hl << 2; 2397 if (mp->m_len < ehdrlen + ip_hlen) 2398 return (FALSE); 2399 ipproto = ip->ip_p; 2400 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2401 break; 2402 case ETHERTYPE_IPV6: 2403 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2404 ip_hlen = sizeof(struct ip6_hdr); 2405 if (mp->m_len < ehdrlen + ip_hlen) 2406 return (FALSE); 2407 ipproto = ip6->ip6_nxt; 2408 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2409 break; 2410 default: 2411 offload = FALSE; 2412 break; 2413 } 2414 2415 vlan_macip_lens |= ip_hlen; 2416 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2417 2418 switch (ipproto) { 2419 case IPPROTO_TCP: 2420 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2421 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2422 break; 2423 2424 case IPPROTO_UDP: 2425 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2426 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2427 break; 2428 2429 #if __FreeBSD_version >= 800000 2430 case IPPROTO_SCTP: 2431 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 2432 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2433 break; 2434 #endif 2435 default: 2436 offload = FALSE; 2437 break; 2438 } 2439 2440 /* Now copy bits into descriptor */ 2441 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2442 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2443 TXD->seqnum_seed = htole32(0); 2444 TXD->mss_l4len_idx = htole32(0); 2445 2446 tx_buffer->m_head = NULL; 2447 tx_buffer->eop_index = -1; 2448 2449 /* We've consumed the first desc, adjust counters */ 2450 if (++ctxd == adapter->num_tx_desc) 2451 ctxd = 0; 2452 txr->next_avail_desc = ctxd; 2453 --txr->tx_avail; 2454 2455 return (offload); 2456 } 2457 2458 /********************************************************************** 2459 * 2460 * Setup work for hardware segmentation offload (TSO) on 2461 * adapters using advanced tx descriptors 2462 * 2463 **********************************************************************/ 2464 static boolean_t 2465 ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2466 { 2467 struct adapter *adapter = txr->adapter; 2468 struct ixgbe_adv_tx_context_desc *TXD; 2469 struct ixv_tx_buf *tx_buffer; 2470 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2471 u32 mss_l4len_idx = 0; 2472 u16 vtag = 0; 2473 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2474 struct ether_vlan_header *eh; 2475 struct ip *ip; 2476 struct tcphdr *th; 2477 2478 2479 /* 2480 * Determine where frame payload starts. 2481 * Jump over vlan headers if already present 2482 */ 2483 eh = mtod(mp, struct ether_vlan_header *); 2484 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2485 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2486 else 2487 ehdrlen = ETHER_HDR_LEN; 2488 2489 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2490 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2491 return FALSE; 2492 2493 ctxd = txr->next_avail_desc; 2494 tx_buffer = &txr->tx_buffers[ctxd]; 2495 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2496 2497 ip = (struct ip *)(mp->m_data + ehdrlen); 2498 if (ip->ip_p != IPPROTO_TCP) 2499 return FALSE; /* 0 */ 2500 ip->ip_sum = 0; 2501 ip_hlen = ip->ip_hl << 2; 2502 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2503 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2504 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2505 tcp_hlen = th->th_off << 2; 2506 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2507 2508 /* This is used in the transmit desc in encap */ 2509 *paylen = mp->m_pkthdr.len - hdrlen; 2510 2511 /* VLAN MACLEN IPLEN */ 2512 if (mp->m_flags & M_VLANTAG) { 2513 vtag = htole16(mp->m_pkthdr.ether_vtag); 2514 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2515 } 2516 2517 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2518 vlan_macip_lens |= ip_hlen; 2519 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2520 2521 /* ADV DTYPE TUCMD */ 2522 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2523 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2524 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2525 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2526 2527 2528 /* MSS L4LEN IDX */ 2529 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2530 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2531 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2532 2533 TXD->seqnum_seed = htole32(0); 2534 tx_buffer->m_head = NULL; 2535 tx_buffer->eop_index = -1; 2536 2537 if (++ctxd == adapter->num_tx_desc) 2538 ctxd = 0; 2539 2540 txr->tx_avail--; 2541 txr->next_avail_desc = ctxd; 2542 return TRUE; 2543 } 2544 2545 2546 /********************************************************************** 2547 * 2548 * Examine each tx_buffer in the used queue. If the hardware is done 2549 * processing the packet then free associated resources. The 2550 * tx_buffer is put back on the free queue. 2551 * 2552 **********************************************************************/ 2553 static boolean_t 2554 ixv_txeof(struct tx_ring *txr) 2555 { 2556 struct adapter *adapter = txr->adapter; 2557 struct ifnet *ifp = adapter->ifp; 2558 u32 first, last, done; 2559 struct ixv_tx_buf *tx_buffer; 2560 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2561 2562 mtx_assert(&txr->tx_mtx, MA_OWNED); 2563 2564 if (txr->tx_avail == adapter->num_tx_desc) 2565 return FALSE; 2566 2567 first = txr->next_to_clean; 2568 tx_buffer = &txr->tx_buffers[first]; 2569 /* For cleanup we just use legacy struct */ 2570 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2571 last = tx_buffer->eop_index; 2572 if (last == -1) 2573 return FALSE; 2574 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2575 2576 /* 2577 ** Get the index of the first descriptor 2578 ** BEYOND the EOP and call that 'done'. 2579 ** I do this so the comparison in the 2580 ** inner while loop below can be simple 2581 */ 2582 if (++last == adapter->num_tx_desc) last = 0; 2583 done = last; 2584 2585 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2586 BUS_DMASYNC_POSTREAD); 2587 /* 2588 ** Only the EOP descriptor of a packet now has the DD 2589 ** bit set, this is what we look for... 2590 */ 2591 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2592 /* We clean the range of the packet */ 2593 while (first != done) { 2594 tx_desc->upper.data = 0; 2595 tx_desc->lower.data = 0; 2596 tx_desc->buffer_addr = 0; 2597 ++txr->tx_avail; 2598 2599 if (tx_buffer->m_head) { 2600 bus_dmamap_sync(txr->txtag, 2601 tx_buffer->map, 2602 BUS_DMASYNC_POSTWRITE); 2603 bus_dmamap_unload(txr->txtag, 2604 tx_buffer->map); 2605 m_freem(tx_buffer->m_head); 2606 tx_buffer->m_head = NULL; 2607 tx_buffer->map = NULL; 2608 } 2609 tx_buffer->eop_index = -1; 2610 txr->watchdog_time = ticks; 2611 2612 if (++first == adapter->num_tx_desc) 2613 first = 0; 2614 2615 tx_buffer = &txr->tx_buffers[first]; 2616 tx_desc = 2617 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2618 } 2619 ++ifp->if_opackets; 2620 /* See if there is more work now */ 2621 last = tx_buffer->eop_index; 2622 if (last != -1) { 2623 eop_desc = 2624 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2625 /* Get next done point */ 2626 if (++last == adapter->num_tx_desc) last = 0; 2627 done = last; 2628 } else 2629 break; 2630 } 2631 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2632 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2633 2634 txr->next_to_clean = first; 2635 2636 /* 2637 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2638 * it is OK to send packets. If there are no pending descriptors, 2639 * clear the timeout. Otherwise, if some descriptors have been freed, 2640 * restart the timeout. 2641 */ 2642 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2643 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2644 if (txr->tx_avail == adapter->num_tx_desc) { 2645 txr->watchdog_check = FALSE; 2646 return FALSE; 2647 } 2648 } 2649 2650 return TRUE; 2651 } 2652 2653 /********************************************************************* 2654 * 2655 * Refresh mbuf buffers for RX descriptor rings 2656 * - now keeps its own state so discards due to resource 2657 * exhaustion are unnecessary, if an mbuf cannot be obtained 2658 * it just returns, keeping its placeholder, thus it can simply 2659 * be recalled to try again. 2660 * 2661 **********************************************************************/ 2662 static void 2663 ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2664 { 2665 struct adapter *adapter = rxr->adapter; 2666 bus_dma_segment_t hseg[1]; 2667 bus_dma_segment_t pseg[1]; 2668 struct ixv_rx_buf *rxbuf; 2669 struct mbuf *mh, *mp; 2670 int i, nsegs, error, cleaned; 2671 2672 i = rxr->next_to_refresh; 2673 cleaned = -1; /* Signify no completions */ 2674 while (i != limit) { 2675 rxbuf = &rxr->rx_buffers[i]; 2676 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2677 mh = m_gethdr(M_DONTWAIT, MT_DATA); 2678 if (mh == NULL) 2679 goto update; 2680 mh->m_pkthdr.len = mh->m_len = MHLEN; 2681 mh->m_len = MHLEN; 2682 mh->m_flags |= M_PKTHDR; 2683 m_adj(mh, ETHER_ALIGN); 2684 /* Get the memory mapping */ 2685 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2686 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 2687 if (error != 0) { 2688 printf("GET BUF: dmamap load" 2689 " failure - %d\n", error); 2690 m_free(mh); 2691 goto update; 2692 } 2693 rxbuf->m_head = mh; 2694 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2695 BUS_DMASYNC_PREREAD); 2696 rxr->rx_base[i].read.hdr_addr = 2697 htole64(hseg[0].ds_addr); 2698 } 2699 2700 if (rxbuf->m_pack == NULL) { 2701 mp = m_getjcl(M_DONTWAIT, MT_DATA, 2702 M_PKTHDR, adapter->rx_mbuf_sz); 2703 if (mp == NULL) 2704 goto update; 2705 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2706 /* Get the memory mapping */ 2707 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2708 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 2709 if (error != 0) { 2710 printf("GET BUF: dmamap load" 2711 " failure - %d\n", error); 2712 m_free(mp); 2713 goto update; 2714 } 2715 rxbuf->m_pack = mp; 2716 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2717 BUS_DMASYNC_PREREAD); 2718 rxr->rx_base[i].read.pkt_addr = 2719 htole64(pseg[0].ds_addr); 2720 } 2721 2722 cleaned = i; 2723 /* Calculate next index */ 2724 if (++i == adapter->num_rx_desc) 2725 i = 0; 2726 /* This is the work marker for refresh */ 2727 rxr->next_to_refresh = i; 2728 } 2729 update: 2730 if (cleaned != -1) /* If we refreshed some, bump tail */ 2731 IXGBE_WRITE_REG(&adapter->hw, 2732 IXGBE_VFRDT(rxr->me), cleaned); 2733 return; 2734 } 2735 2736 /********************************************************************* 2737 * 2738 * Allocate memory for rx_buffer structures. Since we use one 2739 * rx_buffer per received packet, the maximum number of rx_buffer's 2740 * that we'll need is equal to the number of receive descriptors 2741 * that we've allocated. 2742 * 2743 **********************************************************************/ 2744 static int 2745 ixv_allocate_receive_buffers(struct rx_ring *rxr) 2746 { 2747 struct adapter *adapter = rxr->adapter; 2748 device_t dev = adapter->dev; 2749 struct ixv_rx_buf *rxbuf; 2750 int i, bsize, error; 2751 2752 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2753 if (!(rxr->rx_buffers = 2754 (struct ixv_rx_buf *) malloc(bsize, 2755 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2756 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 2757 error = ENOMEM; 2758 goto fail; 2759 } 2760 2761 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2762 1, 0, /* alignment, bounds */ 2763 BUS_SPACE_MAXADDR, /* lowaddr */ 2764 BUS_SPACE_MAXADDR, /* highaddr */ 2765 NULL, NULL, /* filter, filterarg */ 2766 MSIZE, /* maxsize */ 2767 1, /* nsegments */ 2768 MSIZE, /* maxsegsize */ 2769 0, /* flags */ 2770 NULL, /* lockfunc */ 2771 NULL, /* lockfuncarg */ 2772 &rxr->htag))) { 2773 device_printf(dev, "Unable to create RX DMA tag\n"); 2774 goto fail; 2775 } 2776 2777 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2778 1, 0, /* alignment, bounds */ 2779 BUS_SPACE_MAXADDR, /* lowaddr */ 2780 BUS_SPACE_MAXADDR, /* highaddr */ 2781 NULL, NULL, /* filter, filterarg */ 2782 MJUMPAGESIZE, /* maxsize */ 2783 1, /* nsegments */ 2784 MJUMPAGESIZE, /* maxsegsize */ 2785 0, /* flags */ 2786 NULL, /* lockfunc */ 2787 NULL, /* lockfuncarg */ 2788 &rxr->ptag))) { 2789 device_printf(dev, "Unable to create RX DMA tag\n"); 2790 goto fail; 2791 } 2792 2793 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2794 rxbuf = &rxr->rx_buffers[i]; 2795 error = bus_dmamap_create(rxr->htag, 2796 BUS_DMA_NOWAIT, &rxbuf->hmap); 2797 if (error) { 2798 device_printf(dev, "Unable to create RX head map\n"); 2799 goto fail; 2800 } 2801 error = bus_dmamap_create(rxr->ptag, 2802 BUS_DMA_NOWAIT, &rxbuf->pmap); 2803 if (error) { 2804 device_printf(dev, "Unable to create RX pkt map\n"); 2805 goto fail; 2806 } 2807 } 2808 2809 return (0); 2810 2811 fail: 2812 /* Frees all, but can handle partial completion */ 2813 ixv_free_receive_structures(adapter); 2814 return (error); 2815 } 2816 2817 static void 2818 ixv_free_receive_ring(struct rx_ring *rxr) 2819 { 2820 struct adapter *adapter; 2821 struct ixv_rx_buf *rxbuf; 2822 int i; 2823 2824 adapter = rxr->adapter; 2825 for (i = 0; i < adapter->num_rx_desc; i++) { 2826 rxbuf = &rxr->rx_buffers[i]; 2827 if (rxbuf->m_head != NULL) { 2828 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2829 BUS_DMASYNC_POSTREAD); 2830 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 2831 rxbuf->m_head->m_flags |= M_PKTHDR; 2832 m_freem(rxbuf->m_head); 2833 } 2834 if (rxbuf->m_pack != NULL) { 2835 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2836 BUS_DMASYNC_POSTREAD); 2837 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 2838 rxbuf->m_pack->m_flags |= M_PKTHDR; 2839 m_freem(rxbuf->m_pack); 2840 } 2841 rxbuf->m_head = NULL; 2842 rxbuf->m_pack = NULL; 2843 } 2844 } 2845 2846 2847 /********************************************************************* 2848 * 2849 * Initialize a receive ring and its buffers. 2850 * 2851 **********************************************************************/ 2852 static int 2853 ixv_setup_receive_ring(struct rx_ring *rxr) 2854 { 2855 struct adapter *adapter; 2856 struct ifnet *ifp; 2857 device_t dev; 2858 struct ixv_rx_buf *rxbuf; 2859 bus_dma_segment_t pseg[1], hseg[1]; 2860 struct lro_ctrl *lro = &rxr->lro; 2861 int rsize, nsegs, error = 0; 2862 2863 adapter = rxr->adapter; 2864 ifp = adapter->ifp; 2865 dev = adapter->dev; 2866 2867 /* Clear the ring contents */ 2868 IXV_RX_LOCK(rxr); 2869 rsize = roundup2(adapter->num_rx_desc * 2870 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2871 bzero((void *)rxr->rx_base, rsize); 2872 2873 /* Free current RX buffer structs and their mbufs */ 2874 ixv_free_receive_ring(rxr); 2875 2876 /* Configure header split? */ 2877 if (ixv_header_split) 2878 rxr->hdr_split = TRUE; 2879 2880 /* Now replenish the mbufs */ 2881 for (int j = 0; j != adapter->num_rx_desc; ++j) { 2882 struct mbuf *mh, *mp; 2883 2884 rxbuf = &rxr->rx_buffers[j]; 2885 /* 2886 ** Dont allocate mbufs if not 2887 ** doing header split, its wasteful 2888 */ 2889 if (rxr->hdr_split == FALSE) 2890 goto skip_head; 2891 2892 /* First the header */ 2893 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); 2894 if (rxbuf->m_head == NULL) { 2895 error = ENOBUFS; 2896 goto fail; 2897 } 2898 m_adj(rxbuf->m_head, ETHER_ALIGN); 2899 mh = rxbuf->m_head; 2900 mh->m_len = mh->m_pkthdr.len = MHLEN; 2901 mh->m_flags |= M_PKTHDR; 2902 /* Get the memory mapping */ 2903 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2904 rxbuf->hmap, rxbuf->m_head, hseg, 2905 &nsegs, BUS_DMA_NOWAIT); 2906 if (error != 0) /* Nothing elegant to do here */ 2907 goto fail; 2908 bus_dmamap_sync(rxr->htag, 2909 rxbuf->hmap, BUS_DMASYNC_PREREAD); 2910 /* Update descriptor */ 2911 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 2912 2913 skip_head: 2914 /* Now the payload cluster */ 2915 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, 2916 M_PKTHDR, adapter->rx_mbuf_sz); 2917 if (rxbuf->m_pack == NULL) { 2918 error = ENOBUFS; 2919 goto fail; 2920 } 2921 mp = rxbuf->m_pack; 2922 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2923 /* Get the memory mapping */ 2924 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2925 rxbuf->pmap, mp, pseg, 2926 &nsegs, BUS_DMA_NOWAIT); 2927 if (error != 0) 2928 goto fail; 2929 bus_dmamap_sync(rxr->ptag, 2930 rxbuf->pmap, BUS_DMASYNC_PREREAD); 2931 /* Update descriptor */ 2932 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 2933 } 2934 2935 2936 /* Setup our descriptor indices */ 2937 rxr->next_to_check = 0; 2938 rxr->next_to_refresh = 0; 2939 rxr->lro_enabled = FALSE; 2940 rxr->rx_split_packets = 0; 2941 rxr->rx_bytes = 0; 2942 2943 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2944 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2945 2946 /* 2947 ** Now set up the LRO interface: 2948 */ 2949 if (ifp->if_capenable & IFCAP_LRO) { 2950 int err = tcp_lro_init(lro); 2951 if (err) { 2952 device_printf(dev, "LRO Initialization failed!\n"); 2953 goto fail; 2954 } 2955 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 2956 rxr->lro_enabled = TRUE; 2957 lro->ifp = adapter->ifp; 2958 } 2959 2960 IXV_RX_UNLOCK(rxr); 2961 return (0); 2962 2963 fail: 2964 ixv_free_receive_ring(rxr); 2965 IXV_RX_UNLOCK(rxr); 2966 return (error); 2967 } 2968 2969 /********************************************************************* 2970 * 2971 * Initialize all receive rings. 2972 * 2973 **********************************************************************/ 2974 static int 2975 ixv_setup_receive_structures(struct adapter *adapter) 2976 { 2977 struct rx_ring *rxr = adapter->rx_rings; 2978 int j; 2979 2980 for (j = 0; j < adapter->num_queues; j++, rxr++) 2981 if (ixv_setup_receive_ring(rxr)) 2982 goto fail; 2983 2984 return (0); 2985 fail: 2986 /* 2987 * Free RX buffers allocated so far, we will only handle 2988 * the rings that completed, the failing case will have 2989 * cleaned up for itself. 'j' failed, so its the terminus. 2990 */ 2991 for (int i = 0; i < j; ++i) { 2992 rxr = &adapter->rx_rings[i]; 2993 ixv_free_receive_ring(rxr); 2994 } 2995 2996 return (ENOBUFS); 2997 } 2998 2999 /********************************************************************* 3000 * 3001 * Setup receive registers and features. 3002 * 3003 **********************************************************************/ 3004 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3005 3006 static void 3007 ixv_initialize_receive_units(struct adapter *adapter) 3008 { 3009 struct rx_ring *rxr = adapter->rx_rings; 3010 struct ixgbe_hw *hw = &adapter->hw; 3011 struct ifnet *ifp = adapter->ifp; 3012 u32 bufsz, fctrl, rxcsum, hlreg; 3013 3014 3015 /* Enable broadcasts */ 3016 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3017 fctrl |= IXGBE_FCTRL_BAM; 3018 fctrl |= IXGBE_FCTRL_DPF; 3019 fctrl |= IXGBE_FCTRL_PMCF; 3020 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3021 3022 /* Set for Jumbo Frames? */ 3023 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3024 if (ifp->if_mtu > ETHERMTU) { 3025 hlreg |= IXGBE_HLREG0_JUMBOEN; 3026 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3027 } else { 3028 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3029 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3030 } 3031 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3032 3033 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3034 u64 rdba = rxr->rxdma.dma_paddr; 3035 u32 reg, rxdctl; 3036 3037 /* Do the queue enabling first */ 3038 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3039 rxdctl |= IXGBE_RXDCTL_ENABLE; 3040 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3041 for (int k = 0; k < 10; k++) { 3042 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3043 IXGBE_RXDCTL_ENABLE) 3044 break; 3045 else 3046 msec_delay(1); 3047 } 3048 wmb(); 3049 3050 /* Setup the Base and Length of the Rx Descriptor Ring */ 3051 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3052 (rdba & 0x00000000ffffffffULL)); 3053 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3054 (rdba >> 32)); 3055 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3056 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3057 3058 /* Set up the SRRCTL register */ 3059 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3060 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3061 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3062 reg |= bufsz; 3063 if (rxr->hdr_split) { 3064 /* Use a standard mbuf for the header */ 3065 reg |= ((IXV_RX_HDR << 3066 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3067 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3068 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3069 } else 3070 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3071 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3072 3073 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3074 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3075 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3076 adapter->num_rx_desc - 1); 3077 } 3078 3079 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3080 3081 if (ifp->if_capenable & IFCAP_RXCSUM) 3082 rxcsum |= IXGBE_RXCSUM_PCSD; 3083 3084 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3085 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3086 3087 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3088 3089 return; 3090 } 3091 3092 /********************************************************************* 3093 * 3094 * Free all receive rings. 3095 * 3096 **********************************************************************/ 3097 static void 3098 ixv_free_receive_structures(struct adapter *adapter) 3099 { 3100 struct rx_ring *rxr = adapter->rx_rings; 3101 3102 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3103 struct lro_ctrl *lro = &rxr->lro; 3104 ixv_free_receive_buffers(rxr); 3105 /* Free LRO memory */ 3106 tcp_lro_free(lro); 3107 /* Free the ring memory as well */ 3108 ixv_dma_free(adapter, &rxr->rxdma); 3109 } 3110 3111 free(adapter->rx_rings, M_DEVBUF); 3112 } 3113 3114 3115 /********************************************************************* 3116 * 3117 * Free receive ring data structures 3118 * 3119 **********************************************************************/ 3120 static void 3121 ixv_free_receive_buffers(struct rx_ring *rxr) 3122 { 3123 struct adapter *adapter = rxr->adapter; 3124 struct ixv_rx_buf *rxbuf; 3125 3126 INIT_DEBUGOUT("free_receive_structures: begin"); 3127 3128 /* Cleanup any existing buffers */ 3129 if (rxr->rx_buffers != NULL) { 3130 for (int i = 0; i < adapter->num_rx_desc; i++) { 3131 rxbuf = &rxr->rx_buffers[i]; 3132 if (rxbuf->m_head != NULL) { 3133 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3134 BUS_DMASYNC_POSTREAD); 3135 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3136 rxbuf->m_head->m_flags |= M_PKTHDR; 3137 m_freem(rxbuf->m_head); 3138 } 3139 if (rxbuf->m_pack != NULL) { 3140 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3141 BUS_DMASYNC_POSTREAD); 3142 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3143 rxbuf->m_pack->m_flags |= M_PKTHDR; 3144 m_freem(rxbuf->m_pack); 3145 } 3146 rxbuf->m_head = NULL; 3147 rxbuf->m_pack = NULL; 3148 if (rxbuf->hmap != NULL) { 3149 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 3150 rxbuf->hmap = NULL; 3151 } 3152 if (rxbuf->pmap != NULL) { 3153 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3154 rxbuf->pmap = NULL; 3155 } 3156 } 3157 if (rxr->rx_buffers != NULL) { 3158 free(rxr->rx_buffers, M_DEVBUF); 3159 rxr->rx_buffers = NULL; 3160 } 3161 } 3162 3163 if (rxr->htag != NULL) { 3164 bus_dma_tag_destroy(rxr->htag); 3165 rxr->htag = NULL; 3166 } 3167 if (rxr->ptag != NULL) { 3168 bus_dma_tag_destroy(rxr->ptag); 3169 rxr->ptag = NULL; 3170 } 3171 3172 return; 3173 } 3174 3175 static __inline void 3176 ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3177 { 3178 3179 /* 3180 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3181 * should be computed by hardware. Also it should not have VLAN tag in 3182 * ethernet header. 3183 */ 3184 if (rxr->lro_enabled && 3185 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 3186 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3187 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3188 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3189 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3190 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3191 /* 3192 * Send to the stack if: 3193 ** - LRO not enabled, or 3194 ** - no LRO resources, or 3195 ** - lro enqueue fails 3196 */ 3197 if (rxr->lro.lro_cnt != 0) 3198 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3199 return; 3200 } 3201 (*ifp->if_input)(ifp, m); 3202 } 3203 3204 static __inline void 3205 ixv_rx_discard(struct rx_ring *rxr, int i) 3206 { 3207 struct adapter *adapter = rxr->adapter; 3208 struct ixv_rx_buf *rbuf; 3209 struct mbuf *mh, *mp; 3210 3211 rbuf = &rxr->rx_buffers[i]; 3212 if (rbuf->fmp != NULL) /* Partial chain ? */ 3213 m_freem(rbuf->fmp); 3214 3215 mh = rbuf->m_head; 3216 mp = rbuf->m_pack; 3217 3218 /* Reuse loaded DMA map and just update mbuf chain */ 3219 mh->m_len = MHLEN; 3220 mh->m_flags |= M_PKTHDR; 3221 mh->m_next = NULL; 3222 3223 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; 3224 mp->m_data = mp->m_ext.ext_buf; 3225 mp->m_next = NULL; 3226 return; 3227 } 3228 3229 3230 /********************************************************************* 3231 * 3232 * This routine executes in interrupt context. It replenishes 3233 * the mbufs in the descriptor and sends data which has been 3234 * dma'ed into host memory to upper layer. 3235 * 3236 * We loop at most count times if count is > 0, or until done if 3237 * count < 0. 3238 * 3239 * Return TRUE for more work, FALSE for all clean. 3240 *********************************************************************/ 3241 static bool 3242 ixv_rxeof(struct ix_queue *que, int count) 3243 { 3244 struct adapter *adapter = que->adapter; 3245 struct rx_ring *rxr = que->rxr; 3246 struct ifnet *ifp = adapter->ifp; 3247 struct lro_ctrl *lro = &rxr->lro; 3248 struct lro_entry *queued; 3249 int i, nextp, processed = 0; 3250 u32 staterr = 0; 3251 union ixgbe_adv_rx_desc *cur; 3252 struct ixv_rx_buf *rbuf, *nbuf; 3253 3254 IXV_RX_LOCK(rxr); 3255 3256 for (i = rxr->next_to_check; count != 0;) { 3257 struct mbuf *sendmp, *mh, *mp; 3258 u32 rsc, ptype; 3259 u16 hlen, plen, hdr, vtag; 3260 bool eop; 3261 3262 /* Sync the ring. */ 3263 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3264 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3265 3266 cur = &rxr->rx_base[i]; 3267 staterr = le32toh(cur->wb.upper.status_error); 3268 3269 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3270 break; 3271 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3272 break; 3273 3274 count--; 3275 sendmp = NULL; 3276 nbuf = NULL; 3277 rsc = 0; 3278 cur->wb.upper.status_error = 0; 3279 rbuf = &rxr->rx_buffers[i]; 3280 mh = rbuf->m_head; 3281 mp = rbuf->m_pack; 3282 3283 plen = le16toh(cur->wb.upper.length); 3284 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3285 IXGBE_RXDADV_PKTTYPE_MASK; 3286 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3287 vtag = le16toh(cur->wb.upper.vlan); 3288 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3289 3290 /* Make sure all parts of a bad packet are discarded */ 3291 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3292 (rxr->discard)) { 3293 ifp->if_ierrors++; 3294 rxr->rx_discarded++; 3295 if (!eop) 3296 rxr->discard = TRUE; 3297 else 3298 rxr->discard = FALSE; 3299 ixv_rx_discard(rxr, i); 3300 goto next_desc; 3301 } 3302 3303 if (!eop) { 3304 nextp = i + 1; 3305 if (nextp == adapter->num_rx_desc) 3306 nextp = 0; 3307 nbuf = &rxr->rx_buffers[nextp]; 3308 prefetch(nbuf); 3309 } 3310 /* 3311 ** The header mbuf is ONLY used when header 3312 ** split is enabled, otherwise we get normal 3313 ** behavior, ie, both header and payload 3314 ** are DMA'd into the payload buffer. 3315 ** 3316 ** Rather than using the fmp/lmp global pointers 3317 ** we now keep the head of a packet chain in the 3318 ** buffer struct and pass this along from one 3319 ** descriptor to the next, until we get EOP. 3320 */ 3321 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3322 /* This must be an initial descriptor */ 3323 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3324 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3325 if (hlen > IXV_RX_HDR) 3326 hlen = IXV_RX_HDR; 3327 mh->m_len = hlen; 3328 mh->m_flags |= M_PKTHDR; 3329 mh->m_next = NULL; 3330 mh->m_pkthdr.len = mh->m_len; 3331 /* Null buf pointer so it is refreshed */ 3332 rbuf->m_head = NULL; 3333 /* 3334 ** Check the payload length, this 3335 ** could be zero if its a small 3336 ** packet. 3337 */ 3338 if (plen > 0) { 3339 mp->m_len = plen; 3340 mp->m_next = NULL; 3341 mp->m_flags &= ~M_PKTHDR; 3342 mh->m_next = mp; 3343 mh->m_pkthdr.len += mp->m_len; 3344 /* Null buf pointer so it is refreshed */ 3345 rbuf->m_pack = NULL; 3346 rxr->rx_split_packets++; 3347 } 3348 /* 3349 ** Now create the forward 3350 ** chain so when complete 3351 ** we wont have to. 3352 */ 3353 if (eop == 0) { 3354 /* stash the chain head */ 3355 nbuf->fmp = mh; 3356 /* Make forward chain */ 3357 if (plen) 3358 mp->m_next = nbuf->m_pack; 3359 else 3360 mh->m_next = nbuf->m_pack; 3361 } else { 3362 /* Singlet, prepare to send */ 3363 sendmp = mh; 3364 if (staterr & IXGBE_RXD_STAT_VP) { 3365 sendmp->m_pkthdr.ether_vtag = vtag; 3366 sendmp->m_flags |= M_VLANTAG; 3367 } 3368 } 3369 } else { 3370 /* 3371 ** Either no header split, or a 3372 ** secondary piece of a fragmented 3373 ** split packet. 3374 */ 3375 mp->m_len = plen; 3376 /* 3377 ** See if there is a stored head 3378 ** that determines what we are 3379 */ 3380 sendmp = rbuf->fmp; 3381 rbuf->m_pack = rbuf->fmp = NULL; 3382 3383 if (sendmp != NULL) /* secondary frag */ 3384 sendmp->m_pkthdr.len += mp->m_len; 3385 else { 3386 /* first desc of a non-ps chain */ 3387 sendmp = mp; 3388 sendmp->m_flags |= M_PKTHDR; 3389 sendmp->m_pkthdr.len = mp->m_len; 3390 if (staterr & IXGBE_RXD_STAT_VP) { 3391 sendmp->m_pkthdr.ether_vtag = vtag; 3392 sendmp->m_flags |= M_VLANTAG; 3393 } 3394 } 3395 /* Pass the head pointer on */ 3396 if (eop == 0) { 3397 nbuf->fmp = sendmp; 3398 sendmp = NULL; 3399 mp->m_next = nbuf->m_pack; 3400 } 3401 } 3402 ++processed; 3403 /* Sending this frame? */ 3404 if (eop) { 3405 sendmp->m_pkthdr.rcvif = ifp; 3406 ifp->if_ipackets++; 3407 rxr->rx_packets++; 3408 /* capture data for AIM */ 3409 rxr->bytes += sendmp->m_pkthdr.len; 3410 rxr->rx_bytes += sendmp->m_pkthdr.len; 3411 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 3412 ixv_rx_checksum(staterr, sendmp, ptype); 3413 #if __FreeBSD_version >= 800000 3414 sendmp->m_pkthdr.flowid = que->msix; 3415 sendmp->m_flags |= M_FLOWID; 3416 #endif 3417 } 3418 next_desc: 3419 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3420 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3421 3422 /* Advance our pointers to the next descriptor. */ 3423 if (++i == adapter->num_rx_desc) 3424 i = 0; 3425 3426 /* Now send to the stack or do LRO */ 3427 if (sendmp != NULL) 3428 ixv_rx_input(rxr, ifp, sendmp, ptype); 3429 3430 /* Every 8 descriptors we go to refresh mbufs */ 3431 if (processed == 8) { 3432 ixv_refresh_mbufs(rxr, i); 3433 processed = 0; 3434 } 3435 } 3436 3437 /* Refresh any remaining buf structs */ 3438 if (processed != 0) { 3439 ixv_refresh_mbufs(rxr, i); 3440 processed = 0; 3441 } 3442 3443 rxr->next_to_check = i; 3444 3445 /* 3446 * Flush any outstanding LRO work 3447 */ 3448 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3449 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3450 tcp_lro_flush(lro, queued); 3451 } 3452 3453 IXV_RX_UNLOCK(rxr); 3454 3455 /* 3456 ** We still have cleaning to do? 3457 ** Schedule another interrupt if so. 3458 */ 3459 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3460 ixv_rearm_queues(adapter, (u64)(1 << que->msix)); 3461 return (TRUE); 3462 } 3463 3464 return (FALSE); 3465 } 3466 3467 3468 /********************************************************************* 3469 * 3470 * Verify that the hardware indicated that the checksum is valid. 3471 * Inform the stack about the status of checksum so that stack 3472 * doesn't spend time verifying the checksum. 3473 * 3474 *********************************************************************/ 3475 static void 3476 ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 3477 { 3478 u16 status = (u16) staterr; 3479 u8 errors = (u8) (staterr >> 24); 3480 bool sctp = FALSE; 3481 3482 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3483 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3484 sctp = TRUE; 3485 3486 if (status & IXGBE_RXD_STAT_IPCS) { 3487 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3488 /* IP Checksum Good */ 3489 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3490 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3491 3492 } else 3493 mp->m_pkthdr.csum_flags = 0; 3494 } 3495 if (status & IXGBE_RXD_STAT_L4CS) { 3496 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3497 #if __FreeBSD_version >= 800000 3498 if (sctp) 3499 type = CSUM_SCTP_VALID; 3500 #endif 3501 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3502 mp->m_pkthdr.csum_flags |= type; 3503 if (!sctp) 3504 mp->m_pkthdr.csum_data = htons(0xffff); 3505 } 3506 } 3507 return; 3508 } 3509 3510 static void 3511 ixv_setup_vlan_support(struct adapter *adapter) 3512 { 3513 struct ixgbe_hw *hw = &adapter->hw; 3514 u32 ctrl, vid, vfta, retry; 3515 3516 3517 /* 3518 ** We get here thru init_locked, meaning 3519 ** a soft reset, this has already cleared 3520 ** the VFTA and other state, so if there 3521 ** have been no vlan's registered do nothing. 3522 */ 3523 if (adapter->num_vlans == 0) 3524 return; 3525 3526 /* Enable the queues */ 3527 for (int i = 0; i < adapter->num_queues; i++) { 3528 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3529 ctrl |= IXGBE_RXDCTL_VME; 3530 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3531 } 3532 3533 /* 3534 ** A soft reset zero's out the VFTA, so 3535 ** we need to repopulate it now. 3536 */ 3537 for (int i = 0; i < VFTA_SIZE; i++) { 3538 if (ixv_shadow_vfta[i] == 0) 3539 continue; 3540 vfta = ixv_shadow_vfta[i]; 3541 /* 3542 ** Reconstruct the vlan id's 3543 ** based on the bits set in each 3544 ** of the array ints. 3545 */ 3546 for ( int j = 0; j < 32; j++) { 3547 retry = 0; 3548 if ((vfta & (1 << j)) == 0) 3549 continue; 3550 vid = (i * 32) + j; 3551 /* Call the shared code mailbox routine */ 3552 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3553 if (++retry > 5) 3554 break; 3555 } 3556 } 3557 } 3558 } 3559 3560 /* 3561 ** This routine is run via an vlan config EVENT, 3562 ** it enables us to use the HW Filter table since 3563 ** we can get the vlan id. This just creates the 3564 ** entry in the soft version of the VFTA, init will 3565 ** repopulate the real table. 3566 */ 3567 static void 3568 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3569 { 3570 struct adapter *adapter = ifp->if_softc; 3571 u16 index, bit; 3572 3573 if (ifp->if_softc != arg) /* Not our event */ 3574 return; 3575 3576 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3577 return; 3578 3579 index = (vtag >> 5) & 0x7F; 3580 bit = vtag & 0x1F; 3581 ixv_shadow_vfta[index] |= (1 << bit); 3582 ++adapter->num_vlans; 3583 /* Re-init to load the changes */ 3584 ixv_init(adapter); 3585 } 3586 3587 /* 3588 ** This routine is run via an vlan 3589 ** unconfig EVENT, remove our entry 3590 ** in the soft vfta. 3591 */ 3592 static void 3593 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3594 { 3595 struct adapter *adapter = ifp->if_softc; 3596 u16 index, bit; 3597 3598 if (ifp->if_softc != arg) 3599 return; 3600 3601 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3602 return; 3603 3604 index = (vtag >> 5) & 0x7F; 3605 bit = vtag & 0x1F; 3606 ixv_shadow_vfta[index] &= ~(1 << bit); 3607 --adapter->num_vlans; 3608 /* Re-init to load the changes */ 3609 ixv_init(adapter); 3610 } 3611 3612 static void 3613 ixv_enable_intr(struct adapter *adapter) 3614 { 3615 struct ixgbe_hw *hw = &adapter->hw; 3616 struct ix_queue *que = adapter->queues; 3617 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3618 3619 3620 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3621 3622 mask = IXGBE_EIMS_ENABLE_MASK; 3623 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3624 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3625 3626 for (int i = 0; i < adapter->num_queues; i++, que++) 3627 ixv_enable_queue(adapter, que->msix); 3628 3629 IXGBE_WRITE_FLUSH(hw); 3630 3631 return; 3632 } 3633 3634 static void 3635 ixv_disable_intr(struct adapter *adapter) 3636 { 3637 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3638 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3639 IXGBE_WRITE_FLUSH(&adapter->hw); 3640 return; 3641 } 3642 3643 /* 3644 ** Setup the correct IVAR register for a particular MSIX interrupt 3645 ** - entry is the register array entry 3646 ** - vector is the MSIX vector for this queue 3647 ** - type is RX/TX/MISC 3648 */ 3649 static void 3650 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3651 { 3652 struct ixgbe_hw *hw = &adapter->hw; 3653 u32 ivar, index; 3654 3655 vector |= IXGBE_IVAR_ALLOC_VAL; 3656 3657 if (type == -1) { /* MISC IVAR */ 3658 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3659 ivar &= ~0xFF; 3660 ivar |= vector; 3661 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3662 } else { /* RX/TX IVARS */ 3663 index = (16 * (entry & 1)) + (8 * type); 3664 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3665 ivar &= ~(0xFF << index); 3666 ivar |= (vector << index); 3667 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3668 } 3669 } 3670 3671 static void 3672 ixv_configure_ivars(struct adapter *adapter) 3673 { 3674 struct ix_queue *que = adapter->queues; 3675 3676 for (int i = 0; i < adapter->num_queues; i++, que++) { 3677 /* First the RX queue entry */ 3678 ixv_set_ivar(adapter, i, que->msix, 0); 3679 /* ... and the TX */ 3680 ixv_set_ivar(adapter, i, que->msix, 1); 3681 /* Set an initial value in EITR */ 3682 IXGBE_WRITE_REG(&adapter->hw, 3683 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3684 } 3685 3686 /* For the Link interrupt */ 3687 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3688 } 3689 3690 3691 /* 3692 ** Tasklet handler for MSIX MBX interrupts 3693 ** - do outside interrupt since it might sleep 3694 */ 3695 static void 3696 ixv_handle_mbx(void *context) 3697 { 3698 struct adapter *adapter = context; 3699 3700 ixgbe_check_link(&adapter->hw, 3701 &adapter->link_speed, &adapter->link_up, 0); 3702 ixv_update_link_status(adapter); 3703 } 3704 3705 /* 3706 ** The VF stats registers never have a truely virgin 3707 ** starting point, so this routine tries to make an 3708 ** artificial one, marking ground zero on attach as 3709 ** it were. 3710 */ 3711 static void 3712 ixv_save_stats(struct adapter *adapter) 3713 { 3714 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3715 adapter->stats.saved_reset_vfgprc += 3716 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3717 adapter->stats.saved_reset_vfgptc += 3718 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3719 adapter->stats.saved_reset_vfgorc += 3720 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3721 adapter->stats.saved_reset_vfgotc += 3722 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3723 adapter->stats.saved_reset_vfmprc += 3724 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3725 } 3726 } 3727 3728 static void 3729 ixv_init_stats(struct adapter *adapter) 3730 { 3731 struct ixgbe_hw *hw = &adapter->hw; 3732 3733 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3734 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3735 adapter->stats.last_vfgorc |= 3736 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3737 3738 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3739 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3740 adapter->stats.last_vfgotc |= 3741 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3742 3743 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3744 3745 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3746 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3747 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3748 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3749 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3750 } 3751 3752 #define UPDATE_STAT_32(reg, last, count) \ 3753 { \ 3754 u32 current = IXGBE_READ_REG(hw, reg); \ 3755 if (current < last) \ 3756 count += 0x100000000LL; \ 3757 last = current; \ 3758 count &= 0xFFFFFFFF00000000LL; \ 3759 count |= current; \ 3760 } 3761 3762 #define UPDATE_STAT_36(lsb, msb, last, count) \ 3763 { \ 3764 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3765 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3766 u64 current = ((cur_msb << 32) | cur_lsb); \ 3767 if (current < last) \ 3768 count += 0x1000000000LL; \ 3769 last = current; \ 3770 count &= 0xFFFFFFF000000000LL; \ 3771 count |= current; \ 3772 } 3773 3774 /* 3775 ** ixv_update_stats - Update the board statistics counters. 3776 */ 3777 void 3778 ixv_update_stats(struct adapter *adapter) 3779 { 3780 struct ixgbe_hw *hw = &adapter->hw; 3781 3782 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3783 adapter->stats.vfgprc); 3784 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3785 adapter->stats.vfgptc); 3786 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3787 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3788 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3789 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3790 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3791 adapter->stats.vfmprc); 3792 } 3793 3794 /********************************************************************** 3795 * 3796 * This routine is called only when ixgbe_display_debug_stats is enabled. 3797 * This routine provides a way to take a look at important statistics 3798 * maintained by the driver and hardware. 3799 * 3800 **********************************************************************/ 3801 static void 3802 ixv_print_hw_stats(struct adapter * adapter) 3803 { 3804 device_t dev = adapter->dev; 3805 3806 device_printf(dev,"Std Mbuf Failed = %lu\n", 3807 adapter->mbuf_defrag_failed); 3808 device_printf(dev,"Driver dropped packets = %lu\n", 3809 adapter->dropped_pkts); 3810 device_printf(dev, "watchdog timeouts = %ld\n", 3811 adapter->watchdog_events); 3812 3813 device_printf(dev,"Good Packets Rcvd = %llu\n", 3814 (long long)adapter->stats.vfgprc); 3815 device_printf(dev,"Good Packets Xmtd = %llu\n", 3816 (long long)adapter->stats.vfgptc); 3817 device_printf(dev,"TSO Transmissions = %lu\n", 3818 adapter->tso_tx); 3819 3820 } 3821 3822 /********************************************************************** 3823 * 3824 * This routine is called only when em_display_debug_stats is enabled. 3825 * This routine provides a way to take a look at important statistics 3826 * maintained by the driver and hardware. 3827 * 3828 **********************************************************************/ 3829 static void 3830 ixv_print_debug_info(struct adapter *adapter) 3831 { 3832 device_t dev = adapter->dev; 3833 struct ixgbe_hw *hw = &adapter->hw; 3834 struct ix_queue *que = adapter->queues; 3835 struct rx_ring *rxr; 3836 struct tx_ring *txr; 3837 struct lro_ctrl *lro; 3838 3839 device_printf(dev,"Error Byte Count = %u \n", 3840 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 3841 3842 for (int i = 0; i < adapter->num_queues; i++, que++) { 3843 txr = que->txr; 3844 rxr = que->rxr; 3845 lro = &rxr->lro; 3846 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 3847 que->msix, (long)que->irqs); 3848 device_printf(dev,"RX(%d) Packets Received: %lld\n", 3849 rxr->me, (long long)rxr->rx_packets); 3850 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 3851 rxr->me, (long long)rxr->rx_split_packets); 3852 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 3853 rxr->me, (long)rxr->rx_bytes); 3854 device_printf(dev,"RX(%d) LRO Queued= %d\n", 3855 rxr->me, lro->lro_queued); 3856 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 3857 rxr->me, lro->lro_flushed); 3858 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 3859 txr->me, (long)txr->total_packets); 3860 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 3861 txr->me, (long)txr->no_desc_avail); 3862 } 3863 3864 device_printf(dev,"MBX IRQ Handled: %lu\n", 3865 (long)adapter->mbx_irq); 3866 return; 3867 } 3868 3869 static int 3870 ixv_sysctl_stats(SYSCTL_HANDLER_ARGS) 3871 { 3872 int error; 3873 int result; 3874 struct adapter *adapter; 3875 3876 result = -1; 3877 error = sysctl_handle_int(oidp, &result, 0, req); 3878 3879 if (error || !req->newptr) 3880 return (error); 3881 3882 if (result == 1) { 3883 adapter = (struct adapter *) arg1; 3884 ixv_print_hw_stats(adapter); 3885 } 3886 return error; 3887 } 3888 3889 static int 3890 ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 3891 { 3892 int error, result; 3893 struct adapter *adapter; 3894 3895 result = -1; 3896 error = sysctl_handle_int(oidp, &result, 0, req); 3897 3898 if (error || !req->newptr) 3899 return (error); 3900 3901 if (result == 1) { 3902 adapter = (struct adapter *) arg1; 3903 ixv_print_debug_info(adapter); 3904 } 3905 return error; 3906 } 3907 3908 /* 3909 ** Set flow control using sysctl: 3910 ** Flow control values: 3911 ** 0 - off 3912 ** 1 - rx pause 3913 ** 2 - tx pause 3914 ** 3 - full 3915 */ 3916 static int 3917 ixv_set_flowcntl(SYSCTL_HANDLER_ARGS) 3918 { 3919 int error; 3920 struct adapter *adapter; 3921 3922 error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req); 3923 3924 if (error) 3925 return (error); 3926 3927 adapter = (struct adapter *) arg1; 3928 switch (ixv_flow_control) { 3929 case ixgbe_fc_rx_pause: 3930 case ixgbe_fc_tx_pause: 3931 case ixgbe_fc_full: 3932 adapter->hw.fc.requested_mode = ixv_flow_control; 3933 break; 3934 case ixgbe_fc_none: 3935 default: 3936 adapter->hw.fc.requested_mode = ixgbe_fc_none; 3937 } 3938 3939 ixgbe_fc_enable(&adapter->hw, 0); 3940 return error; 3941 } 3942 3943 static void 3944 ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 3945 const char *description, int *limit, int value) 3946 { 3947 *limit = value; 3948 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 3949 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 3950 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 3951 } 3952 3953