1 /****************************************************************************** 2 3 Copyright (c) 2001-2013, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD: head/sys/dev/ixgbe/ixv.c 275358 2014-12-01 11:45:24Z hselasky $*/ 34 /*$NetBSD: ixv.c,v 1.17 2016/06/10 13:27:15 ozaki-r Exp $*/ 35 36 #include "opt_inet.h" 37 #include "opt_inet6.h" 38 39 #include "ixv.h" 40 #include "vlan.h" 41 42 /********************************************************************* 43 * Driver version 44 *********************************************************************/ 45 char ixv_driver_version[] = "1.1.4"; 46 47 /********************************************************************* 48 * PCI Device ID Table 49 * 50 * Used by probe to select devices to load on 51 * Last field stores an index into ixv_strings 52 * Last entry must be all 0s 53 * 54 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 55 *********************************************************************/ 56 57 static ixv_vendor_info_t ixv_vendor_info_array[] = 58 { 59 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 60 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 61 /* required last entry */ 62 {0, 0, 0, 0, 0} 63 }; 64 65 /********************************************************************* 66 * Table of branding strings 67 *********************************************************************/ 68 69 static const char *ixv_strings[] = { 70 "Intel(R) PRO/10GbE Virtual Function Network Driver" 71 }; 72 73 /********************************************************************* 74 * Function prototypes 75 *********************************************************************/ 76 static int ixv_probe(device_t, cfdata_t, void *); 77 static void ixv_attach(device_t, device_t, void *); 78 static int ixv_detach(device_t, int); 79 #if 0 80 static int ixv_shutdown(device_t); 81 #endif 82 #if __FreeBSD_version < 800000 83 static void ixv_start(struct ifnet *); 84 static void ixv_start_locked(struct tx_ring *, struct ifnet *); 85 #else 86 static int ixv_mq_start(struct ifnet *, struct mbuf *); 87 static int ixv_mq_start_locked(struct ifnet *, 88 struct tx_ring *, struct mbuf *); 89 static void ixv_qflush(struct ifnet *); 90 #endif 91 static int ixv_ioctl(struct ifnet *, u_long, void *); 92 static int ixv_init(struct ifnet *); 93 static void ixv_init_locked(struct adapter *); 94 static void ixv_stop(void *); 95 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 96 static int ixv_media_change(struct ifnet *); 97 static void ixv_identify_hardware(struct adapter *); 98 static int ixv_allocate_pci_resources(struct adapter *, 99 const struct pci_attach_args *); 100 static int ixv_allocate_msix(struct adapter *, 101 const struct pci_attach_args *); 102 static int ixv_allocate_queues(struct adapter *); 103 static int ixv_setup_msix(struct adapter *); 104 static void ixv_free_pci_resources(struct adapter *); 105 static void ixv_local_timer(void *); 106 static void ixv_setup_interface(device_t, struct adapter *); 107 static void ixv_config_link(struct adapter *); 108 109 static int ixv_allocate_transmit_buffers(struct tx_ring *); 110 static int ixv_setup_transmit_structures(struct adapter *); 111 static void ixv_setup_transmit_ring(struct tx_ring *); 112 static void ixv_initialize_transmit_units(struct adapter *); 113 static void ixv_free_transmit_structures(struct adapter *); 114 static void ixv_free_transmit_buffers(struct tx_ring *); 115 116 static int ixv_allocate_receive_buffers(struct rx_ring *); 117 static int ixv_setup_receive_structures(struct adapter *); 118 static int ixv_setup_receive_ring(struct rx_ring *); 119 static void ixv_initialize_receive_units(struct adapter *); 120 static void ixv_free_receive_structures(struct adapter *); 121 static void ixv_free_receive_buffers(struct rx_ring *); 122 123 static void ixv_enable_intr(struct adapter *); 124 static void ixv_disable_intr(struct adapter *); 125 static bool ixv_txeof(struct tx_ring *); 126 static bool ixv_rxeof(struct ix_queue *, int); 127 static void ixv_rx_checksum(u32, struct mbuf *, u32, 128 struct ixgbevf_hw_stats *); 129 static void ixv_set_multi(struct adapter *); 130 static void ixv_update_link_status(struct adapter *); 131 static void ixv_refresh_mbufs(struct rx_ring *, int); 132 static int ixv_xmit(struct tx_ring *, struct mbuf *); 133 static int ixv_sysctl_stats(SYSCTLFN_PROTO); 134 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 135 static int ixv_set_flowcntl(SYSCTLFN_PROTO); 136 static int ixv_dma_malloc(struct adapter *, bus_size_t, 137 struct ixv_dma_alloc *, int); 138 static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 139 static void ixv_add_rx_process_limit(struct adapter *, const char *, 140 const char *, int *, int); 141 static u32 ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 142 static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 143 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 144 static void ixv_configure_ivars(struct adapter *); 145 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 146 147 static void ixv_setup_vlan_support(struct adapter *); 148 #if 0 149 static void ixv_register_vlan(void *, struct ifnet *, u16); 150 static void ixv_unregister_vlan(void *, struct ifnet *, u16); 151 #endif 152 153 static void ixv_save_stats(struct adapter *); 154 static void ixv_init_stats(struct adapter *); 155 static void ixv_update_stats(struct adapter *); 156 157 static __inline void ixv_rx_discard(struct rx_ring *, int); 158 static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 159 struct mbuf *, u32); 160 161 /* The MSI/X Interrupt handlers */ 162 static int ixv_msix_que(void *); 163 static int ixv_msix_mbx(void *); 164 165 /* Deferred interrupt tasklets */ 166 static void ixv_handle_que(void *); 167 static void ixv_handle_mbx(void *); 168 169 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 170 static ixv_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 171 172 /********************************************************************* 173 * FreeBSD Device Interface Entry Points 174 *********************************************************************/ 175 176 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 177 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 178 DVF_DETACH_SHUTDOWN); 179 180 # if 0 181 static device_method_t ixv_methods[] = { 182 /* Device interface */ 183 DEVMETHOD(device_probe, ixv_probe), 184 DEVMETHOD(device_attach, ixv_attach), 185 DEVMETHOD(device_detach, ixv_detach), 186 DEVMETHOD(device_shutdown, ixv_shutdown), 187 DEVMETHOD_END 188 }; 189 #endif 190 191 #if 0 192 static driver_t ixv_driver = { 193 "ix", ixv_methods, sizeof(struct adapter), 194 }; 195 196 extern devclass_t ixgbe_devclass; 197 DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 198 MODULE_DEPEND(ixv, pci, 1, 1, 1); 199 MODULE_DEPEND(ixv, ether, 1, 1, 1); 200 #endif 201 202 /* 203 ** TUNEABLE PARAMETERS: 204 */ 205 206 /* 207 ** AIM: Adaptive Interrupt Moderation 208 ** which means that the interrupt rate 209 ** is varied over time based on the 210 ** traffic for that interrupt vector 211 */ 212 static int ixv_enable_aim = FALSE; 213 #define TUNABLE_INT(__x, __y) 214 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 215 216 /* How many packets rxeof tries to clean at a time */ 217 static int ixv_rx_process_limit = 128; 218 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 219 220 /* Flow control setting, default to full */ 221 static int ixv_flow_control = ixgbe_fc_full; 222 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 223 224 /* 225 * Header split: this causes the hardware to DMA 226 * the header into a seperate mbuf from the payload, 227 * it can be a performance win in some workloads, but 228 * in others it actually hurts, its off by default. 229 */ 230 static int ixv_header_split = FALSE; 231 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 232 233 /* 234 ** Number of TX descriptors per ring, 235 ** setting higher than RX as this seems 236 ** the better performing choice. 237 */ 238 static int ixv_txd = DEFAULT_TXD; 239 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 240 241 /* Number of RX descriptors per ring */ 242 static int ixv_rxd = DEFAULT_RXD; 243 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 244 245 /* 246 ** Shadow VFTA table, this is needed because 247 ** the real filter table gets cleared during 248 ** a soft reset and we need to repopulate it. 249 */ 250 static u32 ixv_shadow_vfta[VFTA_SIZE]; 251 252 /* Keep running tab on them for sanity check */ 253 static int ixv_total_ports; 254 255 /********************************************************************* 256 * Device identification routine 257 * 258 * ixv_probe determines if the driver should be loaded on 259 * adapter based on PCI vendor/device id of the adapter. 260 * 261 * return 1 on success, 0 on failure 262 *********************************************************************/ 263 264 static int 265 ixv_probe(device_t dev, cfdata_t cf, void *aux) 266 { 267 const struct pci_attach_args *pa = aux; 268 269 return (ixv_lookup(pa) != NULL) ? 1 : 0; 270 } 271 272 static ixv_vendor_info_t * 273 ixv_lookup(const struct pci_attach_args *pa) 274 { 275 pcireg_t subid; 276 ixv_vendor_info_t *ent; 277 278 INIT_DEBUGOUT("ixv_probe: begin"); 279 280 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 281 return NULL; 282 283 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 284 285 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 286 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 287 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 288 289 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 290 (ent->subvendor_id == 0)) && 291 292 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 293 (ent->subdevice_id == 0))) { 294 ++ixv_total_ports; 295 return ent; 296 } 297 } 298 return NULL; 299 } 300 301 302 static void 303 ixv_sysctl_attach(struct adapter *adapter) 304 { 305 struct sysctllog **log; 306 const struct sysctlnode *rnode, *cnode; 307 device_t dev; 308 309 dev = adapter->dev; 310 log = &adapter->sysctllog; 311 312 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 313 aprint_error_dev(dev, "could not create sysctl root\n"); 314 return; 315 } 316 317 if (sysctl_createv(log, 0, &rnode, &cnode, 318 CTLFLAG_READWRITE, CTLTYPE_INT, 319 "stats", SYSCTL_DESCR("Statistics"), 320 ixv_sysctl_stats, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 321 aprint_error_dev(dev, "could not create sysctl\n"); 322 323 if (sysctl_createv(log, 0, &rnode, &cnode, 324 CTLFLAG_READWRITE, CTLTYPE_INT, 325 "debug", SYSCTL_DESCR("Debug Info"), 326 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 327 aprint_error_dev(dev, "could not create sysctl\n"); 328 329 if (sysctl_createv(log, 0, &rnode, &cnode, 330 CTLFLAG_READWRITE, CTLTYPE_INT, 331 "flow_control", SYSCTL_DESCR("Flow Control"), 332 ixv_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 333 aprint_error_dev(dev, "could not create sysctl\n"); 334 335 /* XXX This is an *instance* sysctl controlling a *global* variable. 336 * XXX It's that way in the FreeBSD driver that this derives from. 337 */ 338 if (sysctl_createv(log, 0, &rnode, &cnode, 339 CTLFLAG_READWRITE, CTLTYPE_INT, 340 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 341 NULL, 0, &ixv_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 342 aprint_error_dev(dev, "could not create sysctl\n"); 343 } 344 345 /********************************************************************* 346 * Device initialization routine 347 * 348 * The attach entry point is called when the driver is being loaded. 349 * This routine identifies the type of hardware, allocates all resources 350 * and initializes the hardware. 351 * 352 * return 0 on success, positive on failure 353 *********************************************************************/ 354 355 static void 356 ixv_attach(device_t parent, device_t dev, void *aux) 357 { 358 struct adapter *adapter; 359 struct ixgbe_hw *hw; 360 int error = 0; 361 ixv_vendor_info_t *ent; 362 const struct pci_attach_args *pa = aux; 363 364 INIT_DEBUGOUT("ixv_attach: begin"); 365 366 /* Allocate, clear, and link in our adapter structure */ 367 adapter = device_private(dev); 368 adapter->dev = adapter->osdep.dev = dev; 369 hw = &adapter->hw; 370 adapter->osdep.pc = pa->pa_pc; 371 adapter->osdep.tag = pa->pa_tag; 372 adapter->osdep.dmat = pa->pa_dmat; 373 adapter->osdep.attached = false; 374 375 ent = ixv_lookup(pa); 376 377 KASSERT(ent != NULL); 378 379 aprint_normal(": %s, Version - %s\n", 380 ixv_strings[ent->index], ixv_driver_version); 381 382 /* Core Lock Init*/ 383 IXV_CORE_LOCK_INIT(adapter, device_xname(dev)); 384 385 /* SYSCTL APIs */ 386 ixv_sysctl_attach(adapter); 387 388 /* Set up the timer callout */ 389 callout_init(&adapter->timer, 0); 390 391 /* Determine hardware revision */ 392 ixv_identify_hardware(adapter); 393 394 /* Do base PCI setup - map BAR0 */ 395 if (ixv_allocate_pci_resources(adapter, pa)) { 396 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 397 error = ENXIO; 398 goto err_out; 399 } 400 401 /* Do descriptor calc and sanity checks */ 402 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 403 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 404 aprint_error_dev(dev, "TXD config issue, using default!\n"); 405 adapter->num_tx_desc = DEFAULT_TXD; 406 } else 407 adapter->num_tx_desc = ixv_txd; 408 409 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 410 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 411 aprint_error_dev(dev, "RXD config issue, using default!\n"); 412 adapter->num_rx_desc = DEFAULT_RXD; 413 } else 414 adapter->num_rx_desc = ixv_rxd; 415 416 /* Allocate our TX/RX Queues */ 417 if (ixv_allocate_queues(adapter)) { 418 error = ENOMEM; 419 goto err_out; 420 } 421 422 /* 423 ** Initialize the shared code: its 424 ** at this point the mac type is set. 425 */ 426 error = ixgbe_init_shared_code(hw); 427 if (error) { 428 aprint_error_dev(dev,"Shared Code Initialization Failure\n"); 429 error = EIO; 430 goto err_late; 431 } 432 433 /* Setup the mailbox */ 434 ixgbe_init_mbx_params_vf(hw); 435 436 ixgbe_reset_hw(hw); 437 438 /* Get Hardware Flow Control setting */ 439 hw->fc.requested_mode = ixgbe_fc_full; 440 hw->fc.pause_time = IXV_FC_PAUSE; 441 hw->fc.low_water[0] = IXV_FC_LO; 442 hw->fc.high_water[0] = IXV_FC_HI; 443 hw->fc.send_xon = TRUE; 444 445 error = ixgbe_init_hw(hw); 446 if (error) { 447 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 448 error = EIO; 449 goto err_late; 450 } 451 452 error = ixv_allocate_msix(adapter, pa); 453 if (error) 454 goto err_late; 455 456 /* Setup OS specific network interface */ 457 ixv_setup_interface(dev, adapter); 458 459 /* Sysctl for limiting the amount of work done in the taskqueue */ 460 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 461 "max number of rx packets to process", &adapter->rx_process_limit, 462 ixv_rx_process_limit); 463 464 /* Do the stats setup */ 465 ixv_save_stats(adapter); 466 ixv_init_stats(adapter); 467 468 /* Register for VLAN events */ 469 #if 0 /* XXX delete after write? */ 470 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 471 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 472 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 473 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 474 #endif 475 476 INIT_DEBUGOUT("ixv_attach: end"); 477 adapter->osdep.attached = true; 478 return; 479 480 err_late: 481 ixv_free_transmit_structures(adapter); 482 ixv_free_receive_structures(adapter); 483 err_out: 484 ixv_free_pci_resources(adapter); 485 return; 486 487 } 488 489 /********************************************************************* 490 * Device removal routine 491 * 492 * The detach entry point is called when the driver is being removed. 493 * This routine stops the adapter and deallocates all the resources 494 * that were allocated for driver operation. 495 * 496 * return 0 on success, positive on failure 497 *********************************************************************/ 498 499 static int 500 ixv_detach(device_t dev, int flags) 501 { 502 struct adapter *adapter = device_private(dev); 503 struct ix_queue *que = adapter->queues; 504 505 INIT_DEBUGOUT("ixv_detach: begin"); 506 if (adapter->osdep.attached == false) 507 return 0; 508 509 #if NVLAN > 0 510 /* Make sure VLANS are not using driver */ 511 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 512 ; /* nothing to do: no VLANs */ 513 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 514 vlan_ifdetach(adapter->ifp); 515 else { 516 aprint_error_dev(dev, "VLANs in use\n"); 517 return EBUSY; 518 } 519 #endif 520 521 IXV_CORE_LOCK(adapter); 522 ixv_stop(adapter); 523 IXV_CORE_UNLOCK(adapter); 524 525 for (int i = 0; i < adapter->num_queues; i++, que++) { 526 softint_disestablish(que->que_si); 527 } 528 529 /* Drain the Link queue */ 530 softint_disestablish(adapter->mbx_si); 531 532 /* Unregister VLAN events */ 533 #if 0 /* XXX msaitoh delete after write? */ 534 if (adapter->vlan_attach != NULL) 535 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 536 if (adapter->vlan_detach != NULL) 537 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 538 #endif 539 540 ether_ifdetach(adapter->ifp); 541 callout_halt(&adapter->timer, NULL); 542 ixv_free_pci_resources(adapter); 543 #if 0 /* XXX the NetBSD port is probably missing something here */ 544 bus_generic_detach(dev); 545 #endif 546 if_detach(adapter->ifp); 547 548 ixv_free_transmit_structures(adapter); 549 ixv_free_receive_structures(adapter); 550 551 IXV_CORE_LOCK_DESTROY(adapter); 552 return (0); 553 } 554 555 /********************************************************************* 556 * 557 * Shutdown entry point 558 * 559 **********************************************************************/ 560 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 561 static int 562 ixv_shutdown(device_t dev) 563 { 564 struct adapter *adapter = device_private(dev); 565 IXV_CORE_LOCK(adapter); 566 ixv_stop(adapter); 567 IXV_CORE_UNLOCK(adapter); 568 return (0); 569 } 570 #endif 571 572 #if __FreeBSD_version < 800000 573 /********************************************************************* 574 * Transmit entry point 575 * 576 * ixv_start is called by the stack to initiate a transmit. 577 * The driver will remain in this routine as long as there are 578 * packets to transmit and transmit resources are available. 579 * In case resources are not available stack is notified and 580 * the packet is requeued. 581 **********************************************************************/ 582 static void 583 ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 584 { 585 int rc; 586 struct mbuf *m_head; 587 struct adapter *adapter = txr->adapter; 588 589 IXV_TX_LOCK_ASSERT(txr); 590 591 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 592 IFF_RUNNING) 593 return; 594 if (!adapter->link_active) 595 return; 596 597 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 598 599 IFQ_POLL(&ifp->if_snd, m_head); 600 if (m_head == NULL) 601 break; 602 603 if ((rc = ixv_xmit(txr, m_head)) == EAGAIN) { 604 ifp->if_flags |= IFF_OACTIVE; 605 break; 606 } 607 IFQ_DEQUEUE(&ifp->if_snd, m_head); 608 if (rc == EFBIG) { 609 struct mbuf *mtmp; 610 611 if ((mtmp = m_defrag(m_head, M_NOWAIT)) != NULL) { 612 m_head = mtmp; 613 rc = ixv_xmit(txr, m_head); 614 if (rc != 0) 615 adapter->efbig2_tx_dma_setup.ev_count++; 616 } else 617 adapter->m_defrag_failed.ev_count++; 618 } 619 if (rc != 0) { 620 m_freem(m_head); 621 continue; 622 } 623 /* Send a copy of the frame to the BPF listener */ 624 bpf_mtap(ifp, m_head); 625 626 /* Set watchdog on */ 627 txr->watchdog_check = TRUE; 628 getmicrotime(&txr->watchdog_time); 629 } 630 return; 631 } 632 633 /* 634 * Legacy TX start - called by the stack, this 635 * always uses the first tx ring, and should 636 * not be used with multiqueue tx enabled. 637 */ 638 static void 639 ixv_start(struct ifnet *ifp) 640 { 641 struct adapter *adapter = ifp->if_softc; 642 struct tx_ring *txr = adapter->tx_rings; 643 644 if (ifp->if_flags & IFF_RUNNING) { 645 IXV_TX_LOCK(txr); 646 ixv_start_locked(txr, ifp); 647 IXV_TX_UNLOCK(txr); 648 } 649 return; 650 } 651 652 #else 653 654 /* 655 ** Multiqueue Transmit driver 656 ** 657 */ 658 static int 659 ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 660 { 661 struct adapter *adapter = ifp->if_softc; 662 struct ix_queue *que; 663 struct tx_ring *txr; 664 int i = 0, err = 0; 665 666 /* Which queue to use */ 667 if ((m->m_flags & M_FLOWID) != 0) 668 i = m->m_pkthdr.flowid % adapter->num_queues; 669 670 txr = &adapter->tx_rings[i]; 671 que = &adapter->queues[i]; 672 673 if (IXV_TX_TRYLOCK(txr)) { 674 err = ixv_mq_start_locked(ifp, txr, m); 675 IXV_TX_UNLOCK(txr); 676 } else { 677 err = drbr_enqueue(ifp, txr->br, m); 678 softint_schedule(que->que_si); 679 } 680 681 return (err); 682 } 683 684 static int 685 ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 686 { 687 struct adapter *adapter = txr->adapter; 688 struct mbuf *next; 689 int enqueued, err = 0; 690 691 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 692 IFF_RUNNING || adapter->link_active == 0) { 693 if (m != NULL) 694 err = drbr_enqueue(ifp, txr->br, m); 695 return (err); 696 } 697 698 /* Do a clean if descriptors are low */ 699 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 700 ixv_txeof(txr); 701 702 enqueued = 0; 703 if (m != NULL) { 704 err = drbr_dequeue(ifp, txr->br, m); 705 if (err) { 706 return (err); 707 } 708 } 709 /* Process the queue */ 710 while ((next = drbr_peek(ifp, txr->br)) != NULL) { 711 if ((err = ixv_xmit(txr, next)) != 0) { 712 if (next != NULL) { 713 drbr_advance(ifp, txr->br); 714 } else { 715 drbr_putback(ifp, txr->br, next); 716 } 717 break; 718 } 719 drbr_advance(ifp, txr->br); 720 enqueued++; 721 ifp->if_obytes += next->m_pkthdr.len; 722 if (next->m_flags & M_MCAST) 723 ifp->if_omcasts++; 724 /* Send a copy of the frame to the BPF listener */ 725 ETHER_BPF_MTAP(ifp, next); 726 if ((ifp->if_flags & IFF_RUNNING) == 0) 727 break; 728 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 729 ifp->if_flags |= IFF_OACTIVE; 730 break; 731 } 732 } 733 734 if (enqueued > 0) { 735 /* Set watchdog on */ 736 txr->watchdog_check = TRUE; 737 getmicrotime(&txr->watchdog_time); 738 } 739 740 return (err); 741 } 742 743 /* 744 ** Flush all ring buffers 745 */ 746 static void 747 ixv_qflush(struct ifnet *ifp) 748 { 749 struct adapter *adapter = ifp->if_softc; 750 struct tx_ring *txr = adapter->tx_rings; 751 struct mbuf *m; 752 753 for (int i = 0; i < adapter->num_queues; i++, txr++) { 754 IXV_TX_LOCK(txr); 755 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 756 m_freem(m); 757 IXV_TX_UNLOCK(txr); 758 } 759 if_qflush(ifp); 760 } 761 762 #endif 763 764 static int 765 ixv_ifflags_cb(struct ethercom *ec) 766 { 767 struct ifnet *ifp = &ec->ec_if; 768 struct adapter *adapter = ifp->if_softc; 769 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 770 771 IXV_CORE_LOCK(adapter); 772 773 if (change != 0) 774 adapter->if_flags = ifp->if_flags; 775 776 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 777 rc = ENETRESET; 778 779 IXV_CORE_UNLOCK(adapter); 780 781 return rc; 782 } 783 784 /********************************************************************* 785 * Ioctl entry point 786 * 787 * ixv_ioctl is called when the user wants to configure the 788 * interface. 789 * 790 * return 0 on success, positive on failure 791 **********************************************************************/ 792 793 static int 794 ixv_ioctl(struct ifnet * ifp, u_long command, void *data) 795 { 796 struct adapter *adapter = ifp->if_softc; 797 struct ifcapreq *ifcr = data; 798 struct ifreq *ifr = (struct ifreq *) data; 799 int error = 0; 800 int l4csum_en; 801 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 802 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 803 804 switch (command) { 805 case SIOCSIFFLAGS: 806 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 807 break; 808 case SIOCADDMULTI: 809 case SIOCDELMULTI: 810 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 811 break; 812 case SIOCSIFMEDIA: 813 case SIOCGIFMEDIA: 814 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 815 break; 816 case SIOCSIFCAP: 817 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 818 break; 819 case SIOCSIFMTU: 820 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 821 break; 822 default: 823 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 824 break; 825 } 826 827 switch (command) { 828 case SIOCSIFMEDIA: 829 case SIOCGIFMEDIA: 830 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 831 case SIOCSIFCAP: 832 /* Layer-4 Rx checksum offload has to be turned on and 833 * off as a unit. 834 */ 835 l4csum_en = ifcr->ifcr_capenable & l4csum; 836 if (l4csum_en != l4csum && l4csum_en != 0) 837 return EINVAL; 838 /*FALLTHROUGH*/ 839 case SIOCADDMULTI: 840 case SIOCDELMULTI: 841 case SIOCSIFFLAGS: 842 case SIOCSIFMTU: 843 default: 844 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 845 return error; 846 if ((ifp->if_flags & IFF_RUNNING) == 0) 847 ; 848 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 849 IXV_CORE_LOCK(adapter); 850 ixv_init_locked(adapter); 851 IXV_CORE_UNLOCK(adapter); 852 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 853 /* 854 * Multicast list has changed; set the hardware filter 855 * accordingly. 856 */ 857 IXV_CORE_LOCK(adapter); 858 ixv_disable_intr(adapter); 859 ixv_set_multi(adapter); 860 ixv_enable_intr(adapter); 861 IXV_CORE_UNLOCK(adapter); 862 } 863 return 0; 864 } 865 } 866 867 /********************************************************************* 868 * Init entry point 869 * 870 * This routine is used in two ways. It is used by the stack as 871 * init entry point in network interface structure. It is also used 872 * by the driver as a hw/sw initialization routine to get to a 873 * consistent state. 874 * 875 * return 0 on success, positive on failure 876 **********************************************************************/ 877 #define IXGBE_MHADD_MFS_SHIFT 16 878 879 static void 880 ixv_init_locked(struct adapter *adapter) 881 { 882 struct ifnet *ifp = adapter->ifp; 883 device_t dev = adapter->dev; 884 struct ixgbe_hw *hw = &adapter->hw; 885 u32 mhadd, gpie; 886 887 INIT_DEBUGOUT("ixv_init: begin"); 888 KASSERT(mutex_owned(&adapter->core_mtx)); 889 hw->adapter_stopped = FALSE; 890 ixgbe_stop_adapter(hw); 891 callout_stop(&adapter->timer); 892 893 /* reprogram the RAR[0] in case user changed it. */ 894 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 895 896 /* Get the latest mac address, User can use a LAA */ 897 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 898 IXGBE_ETH_LENGTH_OF_ADDRESS); 899 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 900 hw->addr_ctrl.rar_used_count = 1; 901 902 /* Prepare transmit descriptors and buffers */ 903 if (ixv_setup_transmit_structures(adapter)) { 904 aprint_error_dev(dev,"Could not setup transmit structures\n"); 905 ixv_stop(adapter); 906 return; 907 } 908 909 ixgbe_reset_hw(hw); 910 ixv_initialize_transmit_units(adapter); 911 912 /* Setup Multicast table */ 913 ixv_set_multi(adapter); 914 915 /* 916 ** Determine the correct mbuf pool 917 ** for doing jumbo/headersplit 918 */ 919 if (ifp->if_mtu > ETHERMTU) 920 adapter->rx_mbuf_sz = MJUMPAGESIZE; 921 else 922 adapter->rx_mbuf_sz = MCLBYTES; 923 924 /* Prepare receive descriptors and buffers */ 925 if (ixv_setup_receive_structures(adapter)) { 926 device_printf(dev,"Could not setup receive structures\n"); 927 ixv_stop(adapter); 928 return; 929 } 930 931 /* Configure RX settings */ 932 ixv_initialize_receive_units(adapter); 933 934 /* Enable Enhanced MSIX mode */ 935 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 936 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 937 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 938 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 939 940 #if 0 /* XXX isn't it required? -- msaitoh */ 941 /* Set the various hardware offload abilities */ 942 ifp->if_hwassist = 0; 943 if (ifp->if_capenable & IFCAP_TSO4) 944 ifp->if_hwassist |= CSUM_TSO; 945 if (ifp->if_capenable & IFCAP_TXCSUM) { 946 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 947 #if __FreeBSD_version >= 800000 948 ifp->if_hwassist |= CSUM_SCTP; 949 #endif 950 } 951 #endif 952 953 /* Set MTU size */ 954 if (ifp->if_mtu > ETHERMTU) { 955 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 956 mhadd &= ~IXGBE_MHADD_MFS_MASK; 957 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 958 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 959 } 960 961 /* Set up VLAN offload and filter */ 962 ixv_setup_vlan_support(adapter); 963 964 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 965 966 /* Set up MSI/X routing */ 967 ixv_configure_ivars(adapter); 968 969 /* Set up auto-mask */ 970 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 971 972 /* Set moderation on the Link interrupt */ 973 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 974 975 /* Stats init */ 976 ixv_init_stats(adapter); 977 978 /* Config/Enable Link */ 979 ixv_config_link(adapter); 980 981 /* And now turn on interrupts */ 982 ixv_enable_intr(adapter); 983 984 /* Now inform the stack we're ready */ 985 ifp->if_flags |= IFF_RUNNING; 986 ifp->if_flags &= ~IFF_OACTIVE; 987 988 return; 989 } 990 991 static int 992 ixv_init(struct ifnet *ifp) 993 { 994 struct adapter *adapter = ifp->if_softc; 995 996 IXV_CORE_LOCK(adapter); 997 ixv_init_locked(adapter); 998 IXV_CORE_UNLOCK(adapter); 999 return 0; 1000 } 1001 1002 1003 /* 1004 ** 1005 ** MSIX Interrupt Handlers and Tasklets 1006 ** 1007 */ 1008 1009 static inline void 1010 ixv_enable_queue(struct adapter *adapter, u32 vector) 1011 { 1012 struct ixgbe_hw *hw = &adapter->hw; 1013 u32 queue = 1 << vector; 1014 u32 mask; 1015 1016 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1017 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 1018 } 1019 1020 static inline void 1021 ixv_disable_queue(struct adapter *adapter, u32 vector) 1022 { 1023 struct ixgbe_hw *hw = &adapter->hw; 1024 u64 queue = (u64)(1 << vector); 1025 u32 mask; 1026 1027 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1028 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 1029 } 1030 1031 static inline void 1032 ixv_rearm_queues(struct adapter *adapter, u64 queues) 1033 { 1034 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1035 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 1036 } 1037 1038 1039 static void 1040 ixv_handle_que(void *context) 1041 { 1042 struct ix_queue *que = context; 1043 struct adapter *adapter = que->adapter; 1044 struct tx_ring *txr = que->txr; 1045 struct ifnet *ifp = adapter->ifp; 1046 bool more; 1047 1048 if (ifp->if_flags & IFF_RUNNING) { 1049 more = ixv_rxeof(que, adapter->rx_process_limit); 1050 IXV_TX_LOCK(txr); 1051 ixv_txeof(txr); 1052 #if __FreeBSD_version >= 800000 1053 if (!drbr_empty(ifp, txr->br)) 1054 ixv_mq_start_locked(ifp, txr, NULL); 1055 #else 1056 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1057 ixv_start_locked(txr, ifp); 1058 #endif 1059 IXV_TX_UNLOCK(txr); 1060 if (more) { 1061 adapter->req.ev_count++; 1062 softint_schedule(que->que_si); 1063 return; 1064 } 1065 } 1066 1067 /* Reenable this interrupt */ 1068 ixv_enable_queue(adapter, que->msix); 1069 return; 1070 } 1071 1072 /********************************************************************* 1073 * 1074 * MSI Queue Interrupt Service routine 1075 * 1076 **********************************************************************/ 1077 int 1078 ixv_msix_que(void *arg) 1079 { 1080 struct ix_queue *que = arg; 1081 struct adapter *adapter = que->adapter; 1082 struct tx_ring *txr = que->txr; 1083 struct rx_ring *rxr = que->rxr; 1084 bool more_tx, more_rx; 1085 u32 newitr = 0; 1086 1087 ixv_disable_queue(adapter, que->msix); 1088 ++que->irqs; 1089 1090 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1091 1092 IXV_TX_LOCK(txr); 1093 more_tx = ixv_txeof(txr); 1094 /* 1095 ** Make certain that if the stack 1096 ** has anything queued the task gets 1097 ** scheduled to handle it. 1098 */ 1099 #if __FreeBSD_version < 800000 1100 if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd)) 1101 #else 1102 if (!drbr_empty(adapter->ifp, txr->br)) 1103 #endif 1104 more_tx = 1; 1105 IXV_TX_UNLOCK(txr); 1106 1107 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1108 1109 /* Do AIM now? */ 1110 1111 if (ixv_enable_aim == FALSE) 1112 goto no_calc; 1113 /* 1114 ** Do Adaptive Interrupt Moderation: 1115 ** - Write out last calculated setting 1116 ** - Calculate based on average size over 1117 ** the last interval. 1118 */ 1119 if (que->eitr_setting) 1120 IXGBE_WRITE_REG(&adapter->hw, 1121 IXGBE_VTEITR(que->msix), 1122 que->eitr_setting); 1123 1124 que->eitr_setting = 0; 1125 1126 /* Idle, do nothing */ 1127 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1128 goto no_calc; 1129 1130 if ((txr->bytes) && (txr->packets)) 1131 newitr = txr->bytes/txr->packets; 1132 if ((rxr->bytes) && (rxr->packets)) 1133 newitr = max(newitr, 1134 (rxr->bytes / rxr->packets)); 1135 newitr += 24; /* account for hardware frame, crc */ 1136 1137 /* set an upper boundary */ 1138 newitr = min(newitr, 3000); 1139 1140 /* Be nice to the mid range */ 1141 if ((newitr > 300) && (newitr < 1200)) 1142 newitr = (newitr / 3); 1143 else 1144 newitr = (newitr / 2); 1145 1146 newitr |= newitr << 16; 1147 1148 /* save for next interrupt */ 1149 que->eitr_setting = newitr; 1150 1151 /* Reset state */ 1152 txr->bytes = 0; 1153 txr->packets = 0; 1154 rxr->bytes = 0; 1155 rxr->packets = 0; 1156 1157 no_calc: 1158 if (more_tx || more_rx) 1159 softint_schedule(que->que_si); 1160 else /* Reenable this interrupt */ 1161 ixv_enable_queue(adapter, que->msix); 1162 return 1; 1163 } 1164 1165 static int 1166 ixv_msix_mbx(void *arg) 1167 { 1168 struct adapter *adapter = arg; 1169 struct ixgbe_hw *hw = &adapter->hw; 1170 u32 reg; 1171 1172 ++adapter->mbx_irq.ev_count; 1173 1174 /* First get the cause */ 1175 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1176 /* Clear interrupt with write */ 1177 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1178 1179 /* Link status change */ 1180 if (reg & IXGBE_EICR_LSC) 1181 softint_schedule(adapter->mbx_si); 1182 1183 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1184 return 1; 1185 } 1186 1187 /********************************************************************* 1188 * 1189 * Media Ioctl callback 1190 * 1191 * This routine is called whenever the user queries the status of 1192 * the interface using ifconfig. 1193 * 1194 **********************************************************************/ 1195 static void 1196 ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1197 { 1198 struct adapter *adapter = ifp->if_softc; 1199 1200 INIT_DEBUGOUT("ixv_media_status: begin"); 1201 IXV_CORE_LOCK(adapter); 1202 ixv_update_link_status(adapter); 1203 1204 ifmr->ifm_status = IFM_AVALID; 1205 ifmr->ifm_active = IFM_ETHER; 1206 1207 if (!adapter->link_active) { 1208 IXV_CORE_UNLOCK(adapter); 1209 return; 1210 } 1211 1212 ifmr->ifm_status |= IFM_ACTIVE; 1213 1214 switch (adapter->link_speed) { 1215 case IXGBE_LINK_SPEED_1GB_FULL: 1216 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1217 break; 1218 case IXGBE_LINK_SPEED_10GB_FULL: 1219 ifmr->ifm_active |= IFM_FDX; 1220 break; 1221 } 1222 1223 IXV_CORE_UNLOCK(adapter); 1224 1225 return; 1226 } 1227 1228 /********************************************************************* 1229 * 1230 * Media Ioctl callback 1231 * 1232 * This routine is called when the user changes speed/duplex using 1233 * media/mediopt option with ifconfig. 1234 * 1235 **********************************************************************/ 1236 static int 1237 ixv_media_change(struct ifnet * ifp) 1238 { 1239 struct adapter *adapter = ifp->if_softc; 1240 struct ifmedia *ifm = &adapter->media; 1241 1242 INIT_DEBUGOUT("ixv_media_change: begin"); 1243 1244 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1245 return (EINVAL); 1246 1247 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1248 case IFM_AUTO: 1249 break; 1250 default: 1251 device_printf(adapter->dev, "Only auto media type\n"); 1252 return (EINVAL); 1253 } 1254 1255 return (0); 1256 } 1257 1258 /********************************************************************* 1259 * 1260 * This routine maps the mbufs to tx descriptors, allowing the 1261 * TX engine to transmit the packets. 1262 * - return 0 on success, positive on failure 1263 * 1264 **********************************************************************/ 1265 1266 static int 1267 ixv_xmit(struct tx_ring *txr, struct mbuf *m_head) 1268 { 1269 struct m_tag *mtag; 1270 struct adapter *adapter = txr->adapter; 1271 struct ethercom *ec = &adapter->osdep.ec; 1272 u32 olinfo_status = 0, cmd_type_len; 1273 u32 paylen = 0; 1274 int i, j, error; 1275 int first, last = 0; 1276 bus_dmamap_t map; 1277 struct ixv_tx_buf *txbuf; 1278 union ixgbe_adv_tx_desc *txd = NULL; 1279 1280 /* Basic descriptor defines */ 1281 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1282 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1283 1284 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1285 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1286 1287 /* 1288 * Important to capture the first descriptor 1289 * used because it will contain the index of 1290 * the one we tell the hardware to report back 1291 */ 1292 first = txr->next_avail_desc; 1293 txbuf = &txr->tx_buffers[first]; 1294 map = txbuf->map; 1295 1296 /* 1297 * Map the packet for DMA. 1298 */ 1299 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1300 m_head, BUS_DMA_NOWAIT); 1301 1302 switch (error) { 1303 case EAGAIN: 1304 adapter->eagain_tx_dma_setup.ev_count++; 1305 return EAGAIN; 1306 case ENOMEM: 1307 adapter->enomem_tx_dma_setup.ev_count++; 1308 return EAGAIN; 1309 case EFBIG: 1310 adapter->efbig_tx_dma_setup.ev_count++; 1311 return error; 1312 case EINVAL: 1313 adapter->einval_tx_dma_setup.ev_count++; 1314 return error; 1315 default: 1316 adapter->other_tx_dma_setup.ev_count++; 1317 return error; 1318 case 0: 1319 break; 1320 } 1321 1322 /* Make certain there are enough descriptors */ 1323 if (map->dm_nsegs > txr->tx_avail - 2) { 1324 txr->no_desc_avail.ev_count++; 1325 /* XXX s/ixgbe/ixv/ */ 1326 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1327 return EAGAIN; 1328 } 1329 1330 /* 1331 ** Set up the appropriate offload context 1332 ** this becomes the first descriptor of 1333 ** a packet. 1334 */ 1335 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1336 if (ixv_tso_setup(txr, m_head, &paylen)) { 1337 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1338 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1339 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1340 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1341 ++adapter->tso_tx.ev_count; 1342 } else { 1343 ++adapter->tso_err.ev_count; 1344 /* XXX unload DMA map! --dyoung -> easy? --msaitoh */ 1345 return (ENXIO); 1346 } 1347 } else 1348 olinfo_status |= ixv_tx_ctx_setup(txr, m_head); 1349 1350 /* Record payload length */ 1351 if (paylen == 0) 1352 olinfo_status |= m_head->m_pkthdr.len << 1353 IXGBE_ADVTXD_PAYLEN_SHIFT; 1354 1355 i = txr->next_avail_desc; 1356 for (j = 0; j < map->dm_nsegs; j++) { 1357 bus_size_t seglen; 1358 bus_addr_t segaddr; 1359 1360 txbuf = &txr->tx_buffers[i]; 1361 txd = &txr->tx_base[i]; 1362 seglen = map->dm_segs[j].ds_len; 1363 segaddr = htole64(map->dm_segs[j].ds_addr); 1364 1365 txd->read.buffer_addr = segaddr; 1366 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1367 cmd_type_len |seglen); 1368 txd->read.olinfo_status = htole32(olinfo_status); 1369 last = i; /* descriptor that will get completion IRQ */ 1370 1371 if (++i == adapter->num_tx_desc) 1372 i = 0; 1373 1374 txbuf->m_head = NULL; 1375 txbuf->eop_index = -1; 1376 } 1377 1378 txd->read.cmd_type_len |= 1379 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1380 txr->tx_avail -= map->dm_nsegs; 1381 txr->next_avail_desc = i; 1382 1383 txbuf->m_head = m_head; 1384 /* Swap the dma map between the first and last descriptor */ 1385 txr->tx_buffers[first].map = txbuf->map; 1386 txbuf->map = map; 1387 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1388 BUS_DMASYNC_PREWRITE); 1389 1390 /* Set the index of the descriptor that will be marked done */ 1391 txbuf = &txr->tx_buffers[first]; 1392 txbuf->eop_index = last; 1393 1394 /* XXX s/ixgbe/ixg/ */ 1395 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1396 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1397 /* 1398 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1399 * hardware that this frame is available to transmit. 1400 */ 1401 ++txr->total_packets.ev_count; 1402 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1403 1404 return 0; 1405 } 1406 1407 1408 /********************************************************************* 1409 * Multicast Update 1410 * 1411 * This routine is called whenever multicast address list is updated. 1412 * 1413 **********************************************************************/ 1414 #define IXGBE_RAR_ENTRIES 16 1415 1416 static void 1417 ixv_set_multi(struct adapter *adapter) 1418 { 1419 struct ether_multi *enm; 1420 struct ether_multistep step; 1421 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1422 u8 *update_ptr; 1423 int mcnt = 0; 1424 struct ethercom *ec = &adapter->osdep.ec; 1425 1426 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1427 1428 ETHER_FIRST_MULTI(step, ec, enm); 1429 while (enm != NULL) { 1430 bcopy(enm->enm_addrlo, 1431 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1432 IXGBE_ETH_LENGTH_OF_ADDRESS); 1433 mcnt++; 1434 /* XXX This might be required --msaitoh */ 1435 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) 1436 break; 1437 ETHER_NEXT_MULTI(step, enm); 1438 } 1439 1440 update_ptr = mta; 1441 1442 ixgbe_update_mc_addr_list(&adapter->hw, 1443 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1444 1445 return; 1446 } 1447 1448 /* 1449 * This is an iterator function now needed by the multicast 1450 * shared code. It simply feeds the shared code routine the 1451 * addresses in the array of ixv_set_multi() one by one. 1452 */ 1453 static u8 * 1454 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1455 { 1456 u8 *addr = *update_ptr; 1457 u8 *newptr; 1458 *vmdq = 0; 1459 1460 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1461 *update_ptr = newptr; 1462 return addr; 1463 } 1464 1465 /********************************************************************* 1466 * Timer routine 1467 * 1468 * This routine checks for link status,updates statistics, 1469 * and runs the watchdog check. 1470 * 1471 **********************************************************************/ 1472 1473 static void 1474 ixv_local_timer1(void *arg) 1475 { 1476 struct adapter *adapter = arg; 1477 device_t dev = adapter->dev; 1478 struct tx_ring *txr = adapter->tx_rings; 1479 int i; 1480 struct timeval now, elapsed; 1481 1482 KASSERT(mutex_owned(&adapter->core_mtx)); 1483 1484 ixv_update_link_status(adapter); 1485 1486 /* Stats Update */ 1487 ixv_update_stats(adapter); 1488 1489 /* 1490 * If the interface has been paused 1491 * then don't do the watchdog check 1492 */ 1493 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1494 goto out; 1495 /* 1496 ** Check for time since any descriptor was cleaned 1497 */ 1498 for (i = 0; i < adapter->num_queues; i++, txr++) { 1499 IXV_TX_LOCK(txr); 1500 if (txr->watchdog_check == FALSE) { 1501 IXV_TX_UNLOCK(txr); 1502 continue; 1503 } 1504 getmicrotime(&now); 1505 timersub(&now, &txr->watchdog_time, &elapsed); 1506 if (tvtohz(&elapsed) > IXV_WATCHDOG) 1507 goto hung; 1508 IXV_TX_UNLOCK(txr); 1509 } 1510 out: 1511 ixv_rearm_queues(adapter, adapter->que_mask); 1512 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1513 return; 1514 1515 hung: 1516 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1517 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1518 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1519 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1520 device_printf(dev,"TX(%d) desc avail = %d," 1521 "Next TX to Clean = %d\n", 1522 txr->me, txr->tx_avail, txr->next_to_clean); 1523 adapter->ifp->if_flags &= ~IFF_RUNNING; 1524 adapter->watchdog_events.ev_count++; 1525 IXV_TX_UNLOCK(txr); 1526 ixv_init_locked(adapter); 1527 } 1528 1529 static void 1530 ixv_local_timer(void *arg) 1531 { 1532 struct adapter *adapter = arg; 1533 1534 IXV_CORE_LOCK(adapter); 1535 ixv_local_timer1(adapter); 1536 IXV_CORE_UNLOCK(adapter); 1537 } 1538 1539 /* 1540 ** Note: this routine updates the OS on the link state 1541 ** the real check of the hardware only happens with 1542 ** a link interrupt. 1543 */ 1544 static void 1545 ixv_update_link_status(struct adapter *adapter) 1546 { 1547 struct ifnet *ifp = adapter->ifp; 1548 struct tx_ring *txr = adapter->tx_rings; 1549 device_t dev = adapter->dev; 1550 1551 1552 if (adapter->link_up){ 1553 if (adapter->link_active == FALSE) { 1554 if (bootverbose) 1555 device_printf(dev,"Link is up %d Gbps %s \n", 1556 ((adapter->link_speed == 128)? 10:1), 1557 "Full Duplex"); 1558 adapter->link_active = TRUE; 1559 if_link_state_change(ifp, LINK_STATE_UP); 1560 } 1561 } else { /* Link down */ 1562 if (adapter->link_active == TRUE) { 1563 if (bootverbose) 1564 device_printf(dev,"Link is Down\n"); 1565 if_link_state_change(ifp, LINK_STATE_DOWN); 1566 adapter->link_active = FALSE; 1567 for (int i = 0; i < adapter->num_queues; 1568 i++, txr++) 1569 txr->watchdog_check = FALSE; 1570 } 1571 } 1572 1573 return; 1574 } 1575 1576 1577 static void 1578 ixv_ifstop(struct ifnet *ifp, int disable) 1579 { 1580 struct adapter *adapter = ifp->if_softc; 1581 1582 IXV_CORE_LOCK(adapter); 1583 ixv_stop(adapter); 1584 IXV_CORE_UNLOCK(adapter); 1585 } 1586 1587 /********************************************************************* 1588 * 1589 * This routine disables all traffic on the adapter by issuing a 1590 * global reset on the MAC and deallocates TX/RX buffers. 1591 * 1592 **********************************************************************/ 1593 1594 static void 1595 ixv_stop(void *arg) 1596 { 1597 struct ifnet *ifp; 1598 struct adapter *adapter = arg; 1599 struct ixgbe_hw *hw = &adapter->hw; 1600 ifp = adapter->ifp; 1601 1602 KASSERT(mutex_owned(&adapter->core_mtx)); 1603 1604 INIT_DEBUGOUT("ixv_stop: begin\n"); 1605 ixv_disable_intr(adapter); 1606 1607 /* Tell the stack that the interface is no longer active */ 1608 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1609 1610 ixgbe_reset_hw(hw); 1611 adapter->hw.adapter_stopped = FALSE; 1612 ixgbe_stop_adapter(hw); 1613 callout_stop(&adapter->timer); 1614 1615 /* reprogram the RAR[0] in case user changed it. */ 1616 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1617 1618 return; 1619 } 1620 1621 1622 /********************************************************************* 1623 * 1624 * Determine hardware revision. 1625 * 1626 **********************************************************************/ 1627 static void 1628 ixv_identify_hardware(struct adapter *adapter) 1629 { 1630 u16 pci_cmd_word; 1631 pcitag_t tag; 1632 pci_chipset_tag_t pc; 1633 pcireg_t subid, id; 1634 struct ixgbe_hw *hw = &adapter->hw; 1635 1636 pc = adapter->osdep.pc; 1637 tag = adapter->osdep.tag; 1638 1639 /* 1640 ** Make sure BUSMASTER is set, on a VM under 1641 ** KVM it may not be and will break things. 1642 */ 1643 pci_cmd_word = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1644 if (!(pci_cmd_word & PCI_COMMAND_MASTER_ENABLE)) { 1645 INIT_DEBUGOUT("Bus Master bit was not set!\n"); 1646 pci_cmd_word |= PCI_COMMAND_MASTER_ENABLE; 1647 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, pci_cmd_word); 1648 } 1649 1650 id = pci_conf_read(pc, tag, PCI_ID_REG); 1651 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 1652 1653 /* Save off the information about this board */ 1654 hw->vendor_id = PCI_VENDOR(id); 1655 hw->device_id = PCI_PRODUCT(id); 1656 hw->revision_id = PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 1657 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 1658 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 1659 1660 return; 1661 } 1662 1663 /********************************************************************* 1664 * 1665 * Setup MSIX Interrupt resources and handlers 1666 * 1667 **********************************************************************/ 1668 static int 1669 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 1670 { 1671 #if !defined(NETBSD_MSI_OR_MSIX) 1672 return 0; 1673 #else 1674 device_t dev = adapter->dev; 1675 struct ix_queue *que = adapter->queues; 1676 int error, rid, vector = 0; 1677 pci_chipset_tag_t pc; 1678 pcitag_t tag; 1679 char intrbuf[PCI_INTRSTR_LEN]; 1680 const char *intrstr = NULL; 1681 kcpuset_t *affinity; 1682 int cpu_id = 0; 1683 1684 pc = adapter->osdep.pc; 1685 tag = adapter->osdep.tag; 1686 1687 if (pci_msix_alloc_exact(pa, 1688 &adapter->osdep.intrs, IXG_MSIX_NINTR) != 0) 1689 return (ENXIO); 1690 1691 kcpuset_create(&affinity, false); 1692 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1693 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 1694 sizeof(intrbuf)); 1695 #ifdef IXV_MPSAFE 1696 pci_intr_setattr(pc, adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 1697 true); 1698 #endif 1699 /* Set the handler function */ 1700 adapter->osdep.ihs[i] = pci_intr_establish(pc, 1701 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que); 1702 if (adapter->osdep.ihs[i] == NULL) { 1703 que->res = NULL; 1704 aprint_error_dev(dev, 1705 "Failed to register QUE handler"); 1706 kcpuset_destroy(affinity); 1707 return (ENXIO); 1708 } 1709 que->msix = vector; 1710 adapter->que_mask |= (u64)(1 << que->msix); 1711 1712 cpu_id = i; 1713 /* Round-robin affinity */ 1714 kcpuset_zero(affinity); 1715 kcpuset_set(affinity, cpu_id % ncpu); 1716 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 1717 NULL); 1718 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 1719 intrstr); 1720 if (error == 0) 1721 aprint_normal(", bound queue %d to cpu %d\n", 1722 i, cpu_id); 1723 else 1724 aprint_normal("\n"); 1725 1726 que->que_si = softint_establish(SOFTINT_NET, ixv_handle_que, 1727 que); 1728 if (que->que_si == NULL) { 1729 aprint_error_dev(dev, 1730 "could not establish software interrupt\n"); 1731 } 1732 } 1733 1734 /* and Mailbox */ 1735 cpu_id++; 1736 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 1737 sizeof(intrbuf)); 1738 #ifdef IXG_MPSAFE 1739 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, true); 1740 #endif 1741 /* Set the mbx handler function */ 1742 adapter->osdep.ihs[vector] = pci_intr_establish(pc, 1743 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter); 1744 if (adapter->osdep.ihs[vector] == NULL) { 1745 adapter->res = NULL; 1746 aprint_error_dev(dev, "Failed to register LINK handler\n"); 1747 kcpuset_destroy(affinity); 1748 return (ENXIO); 1749 } 1750 /* Round-robin affinity */ 1751 kcpuset_zero(affinity); 1752 kcpuset_set(affinity, cpu_id % ncpu); 1753 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity,NULL); 1754 1755 aprint_normal_dev(dev, 1756 "for link, interrupting at %s, ", intrstr); 1757 if (error == 0) { 1758 aprint_normal("affinity to cpu %d\n", cpu_id); 1759 } 1760 adapter->mbxvec = vector; 1761 /* Tasklets for Mailbox */ 1762 adapter->mbx_si = softint_establish(SOFTINT_NET, ixv_handle_mbx, 1763 adapter); 1764 /* 1765 ** Due to a broken design QEMU will fail to properly 1766 ** enable the guest for MSIX unless the vectors in 1767 ** the table are all set up, so we must rewrite the 1768 ** ENABLE in the MSIX control register again at this 1769 ** point to cause it to successfully initialize us. 1770 */ 1771 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1772 int msix_ctrl; 1773 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 1774 rid += PCI_MSIX_CTL; 1775 msix_ctrl = pci_conf_read(pc, tag, rid); 1776 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 1777 pci_conf_write(pc, tag, rid, msix_ctrl); 1778 } 1779 1780 return (0); 1781 #endif 1782 } 1783 1784 /* 1785 * Setup MSIX resources, note that the VF 1786 * device MUST use MSIX, there is no fallback. 1787 */ 1788 static int 1789 ixv_setup_msix(struct adapter *adapter) 1790 { 1791 #if !defined(NETBSD_MSI_OR_MSIX) 1792 return 0; 1793 #else 1794 device_t dev = adapter->dev; 1795 int want, msgs; 1796 1797 /* 1798 ** Want two vectors: one for a queue, 1799 ** plus an additional for mailbox. 1800 */ 1801 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 1802 if (msgs < IXG_MSIX_NINTR) { 1803 aprint_error_dev(dev,"MSIX config error\n"); 1804 return (ENXIO); 1805 } 1806 want = MIN(msgs, IXG_MSIX_NINTR); 1807 1808 adapter->msix_mem = (void *)1; /* XXX */ 1809 aprint_normal_dev(dev, 1810 "Using MSIX interrupts with %d vectors\n", msgs); 1811 return (want); 1812 #endif 1813 } 1814 1815 1816 static int 1817 ixv_allocate_pci_resources(struct adapter *adapter, 1818 const struct pci_attach_args *pa) 1819 { 1820 pcireg_t memtype; 1821 device_t dev = adapter->dev; 1822 bus_addr_t addr; 1823 int flags; 1824 1825 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1826 1827 switch (memtype) { 1828 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1829 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1830 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1831 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1832 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1833 goto map_err; 1834 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1835 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1836 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1837 } 1838 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1839 adapter->osdep.mem_size, flags, 1840 &adapter->osdep.mem_bus_space_handle) != 0) { 1841 map_err: 1842 adapter->osdep.mem_size = 0; 1843 aprint_error_dev(dev, "unable to map BAR0\n"); 1844 return ENXIO; 1845 } 1846 break; 1847 default: 1848 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1849 return ENXIO; 1850 } 1851 1852 adapter->num_queues = 1; 1853 adapter->hw.back = &adapter->osdep; 1854 1855 /* 1856 ** Now setup MSI/X, should 1857 ** return us the number of 1858 ** configured vectors. 1859 */ 1860 adapter->msix = ixv_setup_msix(adapter); 1861 if (adapter->msix == ENXIO) 1862 return (ENXIO); 1863 else 1864 return (0); 1865 } 1866 1867 static void 1868 ixv_free_pci_resources(struct adapter * adapter) 1869 { 1870 #if !defined(NETBSD_MSI_OR_MSIX) 1871 #else 1872 struct ix_queue *que = adapter->queues; 1873 int rid; 1874 1875 /* 1876 ** Release all msix queue resources: 1877 */ 1878 for (int i = 0; i < adapter->num_queues; i++, que++) { 1879 rid = que->msix + 1; 1880 if (que->res != NULL) 1881 pci_intr_disestablish(adapter->osdep.pc, 1882 adapter->osdep.ihs[i]); 1883 } 1884 1885 1886 /* Clean the Legacy or Link interrupt last */ 1887 if (adapter->mbxvec) /* we are doing MSIX */ 1888 rid = adapter->mbxvec + 1; 1889 else 1890 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1891 1892 if (adapter->osdep.ihs[rid] != NULL) 1893 pci_intr_disestablish(adapter->osdep.pc, 1894 adapter->osdep.ihs[rid]); 1895 adapter->osdep.ihs[rid] = NULL; 1896 1897 #if defined(NETBSD_MSI_OR_MSIX) 1898 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1899 adapter->osdep.nintrs); 1900 #endif 1901 1902 if (adapter->osdep.mem_size != 0) { 1903 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1904 adapter->osdep.mem_bus_space_handle, 1905 adapter->osdep.mem_size); 1906 } 1907 1908 #endif 1909 return; 1910 } 1911 1912 /********************************************************************* 1913 * 1914 * Setup networking device structure and register an interface. 1915 * 1916 **********************************************************************/ 1917 static void 1918 ixv_setup_interface(device_t dev, struct adapter *adapter) 1919 { 1920 struct ethercom *ec = &adapter->osdep.ec; 1921 struct ifnet *ifp; 1922 1923 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1924 1925 ifp = adapter->ifp = &ec->ec_if; 1926 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1927 ifp->if_baudrate = 1000000000; 1928 ifp->if_init = ixv_init; 1929 ifp->if_stop = ixv_ifstop; 1930 ifp->if_softc = adapter; 1931 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1932 ifp->if_ioctl = ixv_ioctl; 1933 #if __FreeBSD_version >= 800000 1934 ifp->if_transmit = ixv_mq_start; 1935 ifp->if_qflush = ixv_qflush; 1936 #else 1937 ifp->if_start = ixv_start; 1938 #endif 1939 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1940 1941 if_attach(ifp); 1942 ether_ifattach(ifp, adapter->hw.mac.addr); 1943 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1944 1945 adapter->max_frame_size = 1946 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1947 1948 /* 1949 * Tell the upper layer(s) we support long frames. 1950 */ 1951 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1952 1953 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 1954 ifp->if_capenable = 0; 1955 1956 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 1957 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 1958 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1959 | ETHERCAP_VLAN_MTU; 1960 ec->ec_capenable = ec->ec_capabilities; 1961 1962 /* Don't enable LRO by default */ 1963 ifp->if_capabilities |= IFCAP_LRO; 1964 1965 /* 1966 ** Dont turn this on by default, if vlans are 1967 ** created on another pseudo device (eg. lagg) 1968 ** then vlan events are not passed thru, breaking 1969 ** operation, but with HW FILTER off it works. If 1970 ** using vlans directly on the em driver you can 1971 ** enable this and get full hardware tag filtering. 1972 */ 1973 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1974 1975 /* 1976 * Specify the media types supported by this adapter and register 1977 * callbacks to update media and link information 1978 */ 1979 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1980 ixv_media_status); 1981 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1982 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1983 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1984 1985 return; 1986 } 1987 1988 static void 1989 ixv_config_link(struct adapter *adapter) 1990 { 1991 struct ixgbe_hw *hw = &adapter->hw; 1992 u32 autoneg, err = 0; 1993 1994 if (hw->mac.ops.check_link) 1995 err = hw->mac.ops.check_link(hw, &autoneg, 1996 &adapter->link_up, FALSE); 1997 if (err) 1998 goto out; 1999 2000 if (hw->mac.ops.setup_link) 2001 err = hw->mac.ops.setup_link(hw, 2002 autoneg, adapter->link_up); 2003 out: 2004 return; 2005 } 2006 2007 /******************************************************************** 2008 * Manage DMA'able memory. 2009 *******************************************************************/ 2010 2011 static int 2012 ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 2013 struct ixv_dma_alloc *dma, int mapflags) 2014 { 2015 device_t dev = adapter->dev; 2016 int r, rsegs; 2017 2018 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2019 DBA_ALIGN, 0, /* alignment, bounds */ 2020 size, /* maxsize */ 2021 1, /* nsegments */ 2022 size, /* maxsegsize */ 2023 BUS_DMA_ALLOCNOW, /* flags */ 2024 &dma->dma_tag); 2025 if (r != 0) { 2026 aprint_error_dev(dev, 2027 "ixv_dma_malloc: bus_dma_tag_create failed; error %u\n", r); 2028 goto fail_0; 2029 } 2030 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2031 size, 2032 dma->dma_tag->dt_alignment, 2033 dma->dma_tag->dt_boundary, 2034 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2035 if (r != 0) { 2036 aprint_error_dev(dev, 2037 "%s: bus_dmamem_alloc failed; error %u\n", __func__, r); 2038 goto fail_1; 2039 } 2040 2041 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2042 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2043 if (r != 0) { 2044 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2045 __func__, r); 2046 goto fail_2; 2047 } 2048 2049 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2050 if (r != 0) { 2051 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2052 __func__, r); 2053 goto fail_3; 2054 } 2055 2056 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2057 size, 2058 NULL, 2059 mapflags | BUS_DMA_NOWAIT); 2060 if (r != 0) { 2061 aprint_error_dev(dev,"%s: bus_dmamap_load failed; error %u\n", 2062 __func__, r); 2063 goto fail_4; 2064 } 2065 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2066 dma->dma_size = size; 2067 return 0; 2068 fail_4: 2069 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2070 fail_3: 2071 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2072 fail_2: 2073 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2074 fail_1: 2075 ixgbe_dma_tag_destroy(dma->dma_tag); 2076 fail_0: 2077 dma->dma_tag = NULL; 2078 return (r); 2079 } 2080 2081 static void 2082 ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 2083 { 2084 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2085 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2086 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2087 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2088 ixgbe_dma_tag_destroy(dma->dma_tag); 2089 } 2090 2091 2092 /********************************************************************* 2093 * 2094 * Allocate memory for the transmit and receive rings, and then 2095 * the descriptors associated with each, called only once at attach. 2096 * 2097 **********************************************************************/ 2098 static int 2099 ixv_allocate_queues(struct adapter *adapter) 2100 { 2101 device_t dev = adapter->dev; 2102 struct ix_queue *que; 2103 struct tx_ring *txr; 2104 struct rx_ring *rxr; 2105 int rsize, tsize, error = 0; 2106 int txconf = 0, rxconf = 0; 2107 2108 /* First allocate the top level queue structs */ 2109 if (!(adapter->queues = 2110 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2111 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2112 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2113 error = ENOMEM; 2114 goto fail; 2115 } 2116 2117 /* First allocate the TX ring struct memory */ 2118 if (!(adapter->tx_rings = 2119 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2120 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2121 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2122 error = ENOMEM; 2123 goto tx_fail; 2124 } 2125 2126 /* Next allocate the RX */ 2127 if (!(adapter->rx_rings = 2128 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2129 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2130 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2131 error = ENOMEM; 2132 goto rx_fail; 2133 } 2134 2135 /* For the ring itself */ 2136 tsize = roundup2(adapter->num_tx_desc * 2137 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2138 2139 /* 2140 * Now set up the TX queues, txconf is needed to handle the 2141 * possibility that things fail midcourse and we need to 2142 * undo memory gracefully 2143 */ 2144 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2145 /* Set up some basics */ 2146 txr = &adapter->tx_rings[i]; 2147 txr->adapter = adapter; 2148 txr->me = i; 2149 2150 /* Initialize the TX side lock */ 2151 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2152 device_xname(dev), txr->me); 2153 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2154 2155 if (ixv_dma_malloc(adapter, tsize, 2156 &txr->txdma, BUS_DMA_NOWAIT)) { 2157 aprint_error_dev(dev, 2158 "Unable to allocate TX Descriptor memory\n"); 2159 error = ENOMEM; 2160 goto err_tx_desc; 2161 } 2162 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2163 bzero((void *)txr->tx_base, tsize); 2164 2165 /* Now allocate transmit buffers for the ring */ 2166 if (ixv_allocate_transmit_buffers(txr)) { 2167 aprint_error_dev(dev, 2168 "Critical Failure setting up transmit buffers\n"); 2169 error = ENOMEM; 2170 goto err_tx_desc; 2171 } 2172 #if __FreeBSD_version >= 800000 2173 /* Allocate a buf ring */ 2174 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2175 M_WAITOK, &txr->tx_mtx); 2176 if (txr->br == NULL) { 2177 aprint_error_dev(dev, 2178 "Critical Failure setting up buf ring\n"); 2179 error = ENOMEM; 2180 goto err_tx_desc; 2181 } 2182 #endif 2183 } 2184 2185 /* 2186 * Next the RX queues... 2187 */ 2188 rsize = roundup2(adapter->num_rx_desc * 2189 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2190 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2191 rxr = &adapter->rx_rings[i]; 2192 /* Set up some basics */ 2193 rxr->adapter = adapter; 2194 rxr->me = i; 2195 2196 /* Initialize the RX side lock */ 2197 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2198 device_xname(dev), rxr->me); 2199 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2200 2201 if (ixv_dma_malloc(adapter, rsize, 2202 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2203 aprint_error_dev(dev, 2204 "Unable to allocate RxDescriptor memory\n"); 2205 error = ENOMEM; 2206 goto err_rx_desc; 2207 } 2208 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2209 bzero((void *)rxr->rx_base, rsize); 2210 2211 /* Allocate receive buffers for the ring*/ 2212 if (ixv_allocate_receive_buffers(rxr)) { 2213 aprint_error_dev(dev, 2214 "Critical Failure setting up receive buffers\n"); 2215 error = ENOMEM; 2216 goto err_rx_desc; 2217 } 2218 } 2219 2220 /* 2221 ** Finally set up the queue holding structs 2222 */ 2223 for (int i = 0; i < adapter->num_queues; i++) { 2224 que = &adapter->queues[i]; 2225 que->adapter = adapter; 2226 que->txr = &adapter->tx_rings[i]; 2227 que->rxr = &adapter->rx_rings[i]; 2228 } 2229 2230 return (0); 2231 2232 err_rx_desc: 2233 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2234 ixv_dma_free(adapter, &rxr->rxdma); 2235 err_tx_desc: 2236 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2237 ixv_dma_free(adapter, &txr->txdma); 2238 free(adapter->rx_rings, M_DEVBUF); 2239 rx_fail: 2240 free(adapter->tx_rings, M_DEVBUF); 2241 tx_fail: 2242 free(adapter->queues, M_DEVBUF); 2243 fail: 2244 return (error); 2245 } 2246 2247 2248 /********************************************************************* 2249 * 2250 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2251 * the information needed to transmit a packet on the wire. This is 2252 * called only once at attach, setup is done every reset. 2253 * 2254 **********************************************************************/ 2255 static int 2256 ixv_allocate_transmit_buffers(struct tx_ring *txr) 2257 { 2258 struct adapter *adapter = txr->adapter; 2259 device_t dev = adapter->dev; 2260 struct ixv_tx_buf *txbuf; 2261 int error, i; 2262 2263 /* 2264 * Setup DMA descriptor areas. 2265 */ 2266 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2267 1, 0, /* alignment, bounds */ 2268 IXV_TSO_SIZE, /* maxsize */ 2269 32, /* nsegments */ 2270 PAGE_SIZE, /* maxsegsize */ 2271 0, /* flags */ 2272 &txr->txtag))) { 2273 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 2274 goto fail; 2275 } 2276 2277 if (!(txr->tx_buffers = 2278 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2279 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2280 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 2281 error = ENOMEM; 2282 goto fail; 2283 } 2284 2285 /* Create the descriptor buffer dma maps */ 2286 txbuf = txr->tx_buffers; 2287 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2288 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 2289 if (error != 0) { 2290 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 2291 goto fail; 2292 } 2293 } 2294 2295 return 0; 2296 fail: 2297 /* We free all, it handles case where we are in the middle */ 2298 ixv_free_transmit_structures(adapter); 2299 return (error); 2300 } 2301 2302 /********************************************************************* 2303 * 2304 * Initialize a transmit ring. 2305 * 2306 **********************************************************************/ 2307 static void 2308 ixv_setup_transmit_ring(struct tx_ring *txr) 2309 { 2310 struct adapter *adapter = txr->adapter; 2311 struct ixv_tx_buf *txbuf; 2312 int i; 2313 2314 /* Clear the old ring contents */ 2315 IXV_TX_LOCK(txr); 2316 bzero((void *)txr->tx_base, 2317 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2318 /* Reset indices */ 2319 txr->next_avail_desc = 0; 2320 txr->next_to_clean = 0; 2321 2322 /* Free any existing tx buffers. */ 2323 txbuf = txr->tx_buffers; 2324 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2325 if (txbuf->m_head != NULL) { 2326 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 2327 0, txbuf->m_head->m_pkthdr.len, 2328 BUS_DMASYNC_POSTWRITE); 2329 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 2330 m_freem(txbuf->m_head); 2331 txbuf->m_head = NULL; 2332 } 2333 /* Clear the EOP index */ 2334 txbuf->eop_index = -1; 2335 } 2336 2337 /* Set number of descriptors available */ 2338 txr->tx_avail = adapter->num_tx_desc; 2339 2340 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2341 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2342 IXV_TX_UNLOCK(txr); 2343 } 2344 2345 /********************************************************************* 2346 * 2347 * Initialize all transmit rings. 2348 * 2349 **********************************************************************/ 2350 static int 2351 ixv_setup_transmit_structures(struct adapter *adapter) 2352 { 2353 struct tx_ring *txr = adapter->tx_rings; 2354 2355 for (int i = 0; i < adapter->num_queues; i++, txr++) 2356 ixv_setup_transmit_ring(txr); 2357 2358 return (0); 2359 } 2360 2361 /********************************************************************* 2362 * 2363 * Enable transmit unit. 2364 * 2365 **********************************************************************/ 2366 static void 2367 ixv_initialize_transmit_units(struct adapter *adapter) 2368 { 2369 struct tx_ring *txr = adapter->tx_rings; 2370 struct ixgbe_hw *hw = &adapter->hw; 2371 2372 2373 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2374 u64 tdba = txr->txdma.dma_paddr; 2375 u32 txctrl, txdctl; 2376 2377 /* Set WTHRESH to 8, burst writeback */ 2378 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2379 txdctl |= (8 << 16); 2380 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2381 /* Now enable */ 2382 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2383 txdctl |= IXGBE_TXDCTL_ENABLE; 2384 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2385 2386 /* Set the HW Tx Head and Tail indices */ 2387 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2388 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2389 2390 /* Setup Transmit Descriptor Cmd Settings */ 2391 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2392 txr->watchdog_check = FALSE; 2393 2394 /* Set Ring parameters */ 2395 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2396 (tdba & 0x00000000ffffffffULL)); 2397 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2398 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2399 adapter->num_tx_desc * 2400 sizeof(struct ixgbe_legacy_tx_desc)); 2401 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2402 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2403 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2404 break; 2405 } 2406 2407 return; 2408 } 2409 2410 /********************************************************************* 2411 * 2412 * Free all transmit rings. 2413 * 2414 **********************************************************************/ 2415 static void 2416 ixv_free_transmit_structures(struct adapter *adapter) 2417 { 2418 struct tx_ring *txr = adapter->tx_rings; 2419 2420 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2421 ixv_free_transmit_buffers(txr); 2422 ixv_dma_free(adapter, &txr->txdma); 2423 IXV_TX_LOCK_DESTROY(txr); 2424 } 2425 free(adapter->tx_rings, M_DEVBUF); 2426 } 2427 2428 /********************************************************************* 2429 * 2430 * Free transmit ring related data structures. 2431 * 2432 **********************************************************************/ 2433 static void 2434 ixv_free_transmit_buffers(struct tx_ring *txr) 2435 { 2436 struct adapter *adapter = txr->adapter; 2437 struct ixv_tx_buf *tx_buffer; 2438 int i; 2439 2440 INIT_DEBUGOUT("free_transmit_ring: begin"); 2441 2442 if (txr->tx_buffers == NULL) 2443 return; 2444 2445 tx_buffer = txr->tx_buffers; 2446 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2447 if (tx_buffer->m_head != NULL) { 2448 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 2449 0, tx_buffer->m_head->m_pkthdr.len, 2450 BUS_DMASYNC_POSTWRITE); 2451 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2452 m_freem(tx_buffer->m_head); 2453 tx_buffer->m_head = NULL; 2454 if (tx_buffer->map != NULL) { 2455 ixgbe_dmamap_destroy(txr->txtag, 2456 tx_buffer->map); 2457 tx_buffer->map = NULL; 2458 } 2459 } else if (tx_buffer->map != NULL) { 2460 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2461 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 2462 tx_buffer->map = NULL; 2463 } 2464 } 2465 #if __FreeBSD_version >= 800000 2466 if (txr->br != NULL) 2467 buf_ring_free(txr->br, M_DEVBUF); 2468 #endif 2469 if (txr->tx_buffers != NULL) { 2470 free(txr->tx_buffers, M_DEVBUF); 2471 txr->tx_buffers = NULL; 2472 } 2473 if (txr->txtag != NULL) { 2474 ixgbe_dma_tag_destroy(txr->txtag); 2475 txr->txtag = NULL; 2476 } 2477 return; 2478 } 2479 2480 /********************************************************************* 2481 * 2482 * Advanced Context Descriptor setup for VLAN or CSUM 2483 * 2484 **********************************************************************/ 2485 2486 static u32 2487 ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2488 { 2489 struct m_tag *mtag; 2490 struct adapter *adapter = txr->adapter; 2491 struct ethercom *ec = &adapter->osdep.ec; 2492 struct ixgbe_adv_tx_context_desc *TXD; 2493 struct ixv_tx_buf *tx_buffer; 2494 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2495 struct ether_vlan_header *eh; 2496 struct ip ip; 2497 struct ip6_hdr ip6; 2498 int ehdrlen, ip_hlen = 0; 2499 u16 etype; 2500 u8 ipproto __diagused = 0; 2501 bool offload; 2502 int ctxd = txr->next_avail_desc; 2503 u16 vtag = 0; 2504 2505 2506 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 2507 2508 tx_buffer = &txr->tx_buffers[ctxd]; 2509 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2510 2511 /* 2512 ** In advanced descriptors the vlan tag must 2513 ** be placed into the descriptor itself. 2514 */ 2515 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 2516 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 2517 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2518 } else if (!offload) 2519 return 0; 2520 2521 /* 2522 * Determine where frame payload starts. 2523 * Jump over vlan headers if already present, 2524 * helpful for QinQ too. 2525 */ 2526 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 2527 eh = mtod(mp, struct ether_vlan_header *); 2528 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2529 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 2530 etype = ntohs(eh->evl_proto); 2531 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2532 } else { 2533 etype = ntohs(eh->evl_encap_proto); 2534 ehdrlen = ETHER_HDR_LEN; 2535 } 2536 2537 /* Set the ether header length */ 2538 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2539 2540 switch (etype) { 2541 case ETHERTYPE_IP: 2542 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 2543 ip_hlen = ip.ip_hl << 2; 2544 ipproto = ip.ip_p; 2545 #if 0 2546 ip.ip_sum = 0; 2547 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 2548 #else 2549 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 2550 ip.ip_sum == 0); 2551 #endif 2552 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2553 break; 2554 case ETHERTYPE_IPV6: 2555 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 2556 ip_hlen = sizeof(ip6); 2557 ipproto = ip6.ip6_nxt; 2558 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2559 break; 2560 default: 2561 break; 2562 } 2563 2564 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 2565 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 2566 2567 vlan_macip_lens |= ip_hlen; 2568 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2569 2570 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 2571 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2572 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 2573 KASSERT(ipproto == IPPROTO_TCP); 2574 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 2575 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2576 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 2577 KASSERT(ipproto == IPPROTO_UDP); 2578 } 2579 2580 /* Now copy bits into descriptor */ 2581 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2582 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2583 TXD->seqnum_seed = htole32(0); 2584 TXD->mss_l4len_idx = htole32(0); 2585 2586 tx_buffer->m_head = NULL; 2587 tx_buffer->eop_index = -1; 2588 2589 /* We've consumed the first desc, adjust counters */ 2590 if (++ctxd == adapter->num_tx_desc) 2591 ctxd = 0; 2592 txr->next_avail_desc = ctxd; 2593 --txr->tx_avail; 2594 2595 return olinfo; 2596 } 2597 2598 /********************************************************************** 2599 * 2600 * Setup work for hardware segmentation offload (TSO) on 2601 * adapters using advanced tx descriptors 2602 * 2603 **********************************************************************/ 2604 static bool 2605 ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2606 { 2607 struct m_tag *mtag; 2608 struct adapter *adapter = txr->adapter; 2609 struct ethercom *ec = &adapter->osdep.ec; 2610 struct ixgbe_adv_tx_context_desc *TXD; 2611 struct ixv_tx_buf *tx_buffer; 2612 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2613 u32 mss_l4len_idx = 0; 2614 u16 vtag = 0; 2615 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2616 struct ether_vlan_header *eh; 2617 struct ip *ip; 2618 struct tcphdr *th; 2619 2620 2621 /* 2622 * Determine where frame payload starts. 2623 * Jump over vlan headers if already present 2624 */ 2625 eh = mtod(mp, struct ether_vlan_header *); 2626 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2627 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2628 else 2629 ehdrlen = ETHER_HDR_LEN; 2630 2631 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2632 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2633 return FALSE; 2634 2635 ctxd = txr->next_avail_desc; 2636 tx_buffer = &txr->tx_buffers[ctxd]; 2637 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2638 2639 ip = (struct ip *)(mp->m_data + ehdrlen); 2640 if (ip->ip_p != IPPROTO_TCP) 2641 return FALSE; /* 0 */ 2642 ip->ip_sum = 0; 2643 ip_hlen = ip->ip_hl << 2; 2644 th = (struct tcphdr *)((char *)ip + ip_hlen); 2645 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 2646 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 2647 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2648 tcp_hlen = th->th_off << 2; 2649 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2650 2651 /* This is used in the transmit desc in encap */ 2652 *paylen = mp->m_pkthdr.len - hdrlen; 2653 2654 /* VLAN MACLEN IPLEN */ 2655 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 2656 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 2657 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2658 } 2659 2660 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2661 vlan_macip_lens |= ip_hlen; 2662 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2663 2664 /* ADV DTYPE TUCMD */ 2665 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2666 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2667 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2668 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2669 2670 2671 /* MSS L4LEN IDX */ 2672 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 2673 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2674 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2675 2676 TXD->seqnum_seed = htole32(0); 2677 tx_buffer->m_head = NULL; 2678 tx_buffer->eop_index = -1; 2679 2680 if (++ctxd == adapter->num_tx_desc) 2681 ctxd = 0; 2682 2683 txr->tx_avail--; 2684 txr->next_avail_desc = ctxd; 2685 return TRUE; 2686 } 2687 2688 2689 /********************************************************************** 2690 * 2691 * Examine each tx_buffer in the used queue. If the hardware is done 2692 * processing the packet then free associated resources. The 2693 * tx_buffer is put back on the free queue. 2694 * 2695 **********************************************************************/ 2696 static bool 2697 ixv_txeof(struct tx_ring *txr) 2698 { 2699 struct adapter *adapter = txr->adapter; 2700 struct ifnet *ifp = adapter->ifp; 2701 u32 first, last, done; 2702 struct ixv_tx_buf *tx_buffer; 2703 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2704 2705 KASSERT(mutex_owned(&txr->tx_mtx)); 2706 2707 if (txr->tx_avail == adapter->num_tx_desc) 2708 return false; 2709 2710 first = txr->next_to_clean; 2711 tx_buffer = &txr->tx_buffers[first]; 2712 /* For cleanup we just use legacy struct */ 2713 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2714 last = tx_buffer->eop_index; 2715 if (last == -1) 2716 return false; 2717 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2718 2719 /* 2720 ** Get the index of the first descriptor 2721 ** BEYOND the EOP and call that 'done'. 2722 ** I do this so the comparison in the 2723 ** inner while loop below can be simple 2724 */ 2725 if (++last == adapter->num_tx_desc) last = 0; 2726 done = last; 2727 2728 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2729 BUS_DMASYNC_POSTREAD); 2730 /* 2731 ** Only the EOP descriptor of a packet now has the DD 2732 ** bit set, this is what we look for... 2733 */ 2734 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2735 /* We clean the range of the packet */ 2736 while (first != done) { 2737 tx_desc->upper.data = 0; 2738 tx_desc->lower.data = 0; 2739 tx_desc->buffer_addr = 0; 2740 ++txr->tx_avail; 2741 2742 if (tx_buffer->m_head) { 2743 bus_dmamap_sync(txr->txtag->dt_dmat, 2744 tx_buffer->map, 2745 0, tx_buffer->m_head->m_pkthdr.len, 2746 BUS_DMASYNC_POSTWRITE); 2747 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2748 m_freem(tx_buffer->m_head); 2749 tx_buffer->m_head = NULL; 2750 tx_buffer->map = NULL; 2751 } 2752 tx_buffer->eop_index = -1; 2753 getmicrotime(&txr->watchdog_time); 2754 2755 if (++first == adapter->num_tx_desc) 2756 first = 0; 2757 2758 tx_buffer = &txr->tx_buffers[first]; 2759 tx_desc = 2760 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2761 } 2762 ++ifp->if_opackets; 2763 /* See if there is more work now */ 2764 last = tx_buffer->eop_index; 2765 if (last != -1) { 2766 eop_desc = 2767 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2768 /* Get next done point */ 2769 if (++last == adapter->num_tx_desc) last = 0; 2770 done = last; 2771 } else 2772 break; 2773 } 2774 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2775 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2776 2777 txr->next_to_clean = first; 2778 2779 /* 2780 * If we have enough room, clear IFF_OACTIVE to tell the stack that 2781 * it is OK to send packets. If there are no pending descriptors, 2782 * clear the timeout. Otherwise, if some descriptors have been freed, 2783 * restart the timeout. 2784 */ 2785 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2786 ifp->if_flags &= ~IFF_OACTIVE; 2787 if (txr->tx_avail == adapter->num_tx_desc) { 2788 txr->watchdog_check = FALSE; 2789 return false; 2790 } 2791 } 2792 2793 return true; 2794 } 2795 2796 /********************************************************************* 2797 * 2798 * Refresh mbuf buffers for RX descriptor rings 2799 * - now keeps its own state so discards due to resource 2800 * exhaustion are unnecessary, if an mbuf cannot be obtained 2801 * it just returns, keeping its placeholder, thus it can simply 2802 * be recalled to try again. 2803 * 2804 **********************************************************************/ 2805 static void 2806 ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2807 { 2808 struct adapter *adapter = rxr->adapter; 2809 struct ixv_rx_buf *rxbuf; 2810 struct mbuf *mh, *mp; 2811 int i, j, error; 2812 bool refreshed = false; 2813 2814 i = j = rxr->next_to_refresh; 2815 /* Get the control variable, one beyond refresh point */ 2816 if (++j == adapter->num_rx_desc) 2817 j = 0; 2818 while (j != limit) { 2819 rxbuf = &rxr->rx_buffers[i]; 2820 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2821 mh = m_gethdr(M_NOWAIT, MT_DATA); 2822 if (mh == NULL) 2823 goto update; 2824 mh->m_pkthdr.len = mh->m_len = MHLEN; 2825 mh->m_flags |= M_PKTHDR; 2826 m_adj(mh, ETHER_ALIGN); 2827 /* Get the memory mapping */ 2828 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 2829 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 2830 if (error != 0) { 2831 printf("GET BUF: dmamap load" 2832 " failure - %d\n", error); 2833 m_free(mh); 2834 goto update; 2835 } 2836 rxbuf->m_head = mh; 2837 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 2838 BUS_DMASYNC_PREREAD); 2839 rxr->rx_base[i].read.hdr_addr = 2840 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 2841 } 2842 2843 if (rxbuf->m_pack == NULL) { 2844 mp = ixgbe_getjcl(&adapter->jcl_head, M_NOWAIT, 2845 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 2846 if (mp == NULL) { 2847 rxr->no_jmbuf.ev_count++; 2848 goto update; 2849 } else 2850 mp = rxbuf->m_pack; 2851 2852 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2853 /* Get the memory mapping */ 2854 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 2855 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 2856 if (error != 0) { 2857 printf("GET BUF: dmamap load" 2858 " failure - %d\n", error); 2859 m_free(mp); 2860 rxbuf->m_pack = NULL; 2861 goto update; 2862 } 2863 rxbuf->m_pack = mp; 2864 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 2865 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 2866 rxr->rx_base[i].read.pkt_addr = 2867 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 2868 } 2869 2870 refreshed = true; 2871 rxr->next_to_refresh = i = j; 2872 /* Calculate next index */ 2873 if (++j == adapter->num_rx_desc) 2874 j = 0; 2875 } 2876 update: 2877 if (refreshed) /* update tail index */ 2878 IXGBE_WRITE_REG(&adapter->hw, 2879 IXGBE_VFRDT(rxr->me), rxr->next_to_refresh); 2880 return; 2881 } 2882 2883 /********************************************************************* 2884 * 2885 * Allocate memory for rx_buffer structures. Since we use one 2886 * rx_buffer per received packet, the maximum number of rx_buffer's 2887 * that we'll need is equal to the number of receive descriptors 2888 * that we've allocated. 2889 * 2890 **********************************************************************/ 2891 static int 2892 ixv_allocate_receive_buffers(struct rx_ring *rxr) 2893 { 2894 struct adapter *adapter = rxr->adapter; 2895 device_t dev = adapter->dev; 2896 struct ixv_rx_buf *rxbuf; 2897 int i, bsize, error; 2898 2899 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2900 if (!(rxr->rx_buffers = 2901 (struct ixv_rx_buf *) malloc(bsize, 2902 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2903 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 2904 error = ENOMEM; 2905 goto fail; 2906 } 2907 2908 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2909 1, 0, /* alignment, bounds */ 2910 MSIZE, /* maxsize */ 2911 1, /* nsegments */ 2912 MSIZE, /* maxsegsize */ 2913 0, /* flags */ 2914 &rxr->htag))) { 2915 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 2916 goto fail; 2917 } 2918 2919 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2920 1, 0, /* alignment, bounds */ 2921 MJUMPAGESIZE, /* maxsize */ 2922 1, /* nsegments */ 2923 MJUMPAGESIZE, /* maxsegsize */ 2924 0, /* flags */ 2925 &rxr->ptag))) { 2926 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 2927 goto fail; 2928 } 2929 2930 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2931 rxbuf = &rxr->rx_buffers[i]; 2932 error = ixgbe_dmamap_create(rxr->htag, 2933 BUS_DMA_NOWAIT, &rxbuf->hmap); 2934 if (error) { 2935 aprint_error_dev(dev, "Unable to create RX head map\n"); 2936 goto fail; 2937 } 2938 error = ixgbe_dmamap_create(rxr->ptag, 2939 BUS_DMA_NOWAIT, &rxbuf->pmap); 2940 if (error) { 2941 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 2942 goto fail; 2943 } 2944 } 2945 2946 return (0); 2947 2948 fail: 2949 /* Frees all, but can handle partial completion */ 2950 ixv_free_receive_structures(adapter); 2951 return (error); 2952 } 2953 2954 static void 2955 ixv_free_receive_ring(struct rx_ring *rxr) 2956 { 2957 struct adapter *adapter; 2958 struct ixv_rx_buf *rxbuf; 2959 int i; 2960 2961 adapter = rxr->adapter; 2962 for (i = 0; i < adapter->num_rx_desc; i++) { 2963 rxbuf = &rxr->rx_buffers[i]; 2964 if (rxbuf->m_head != NULL) { 2965 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 2966 BUS_DMASYNC_POSTREAD); 2967 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 2968 rxbuf->m_head->m_flags |= M_PKTHDR; 2969 m_freem(rxbuf->m_head); 2970 } 2971 if (rxbuf->m_pack != NULL) { 2972 /* XXX not ixgbe_ ? */ 2973 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 2974 0, rxbuf->m_pack->m_pkthdr.len, 2975 BUS_DMASYNC_POSTREAD); 2976 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 2977 rxbuf->m_pack->m_flags |= M_PKTHDR; 2978 m_freem(rxbuf->m_pack); 2979 } 2980 rxbuf->m_head = NULL; 2981 rxbuf->m_pack = NULL; 2982 } 2983 } 2984 2985 2986 /********************************************************************* 2987 * 2988 * Initialize a receive ring and its buffers. 2989 * 2990 **********************************************************************/ 2991 static int 2992 ixv_setup_receive_ring(struct rx_ring *rxr) 2993 { 2994 struct adapter *adapter; 2995 struct ixv_rx_buf *rxbuf; 2996 #ifdef LRO 2997 struct ifnet *ifp; 2998 struct lro_ctrl *lro = &rxr->lro; 2999 #endif /* LRO */ 3000 int rsize, error = 0; 3001 3002 adapter = rxr->adapter; 3003 #ifdef LRO 3004 ifp = adapter->ifp; 3005 #endif /* LRO */ 3006 3007 /* Clear the ring contents */ 3008 IXV_RX_LOCK(rxr); 3009 rsize = roundup2(adapter->num_rx_desc * 3010 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3011 bzero((void *)rxr->rx_base, rsize); 3012 3013 /* Free current RX buffer structs and their mbufs */ 3014 ixv_free_receive_ring(rxr); 3015 3016 IXV_RX_UNLOCK(rxr); 3017 3018 /* Now reinitialize our supply of jumbo mbufs. The number 3019 * or size of jumbo mbufs may have changed. 3020 */ 3021 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 3022 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 3023 3024 IXV_RX_LOCK(rxr); 3025 3026 /* Configure header split? */ 3027 if (ixv_header_split) 3028 rxr->hdr_split = TRUE; 3029 3030 /* Now replenish the mbufs */ 3031 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3032 struct mbuf *mh, *mp; 3033 3034 rxbuf = &rxr->rx_buffers[j]; 3035 /* 3036 ** Dont allocate mbufs if not 3037 ** doing header split, its wasteful 3038 */ 3039 if (rxr->hdr_split == FALSE) 3040 goto skip_head; 3041 3042 /* First the header */ 3043 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3044 if (rxbuf->m_head == NULL) { 3045 error = ENOBUFS; 3046 goto fail; 3047 } 3048 m_adj(rxbuf->m_head, ETHER_ALIGN); 3049 mh = rxbuf->m_head; 3050 mh->m_len = mh->m_pkthdr.len = MHLEN; 3051 mh->m_flags |= M_PKTHDR; 3052 /* Get the memory mapping */ 3053 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3054 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 3055 if (error != 0) /* Nothing elegant to do here */ 3056 goto fail; 3057 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 3058 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3059 /* Update descriptor */ 3060 rxr->rx_base[j].read.hdr_addr = 3061 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3062 3063 skip_head: 3064 /* Now the payload cluster */ 3065 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3066 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3067 if (rxbuf->m_pack == NULL) { 3068 error = ENOBUFS; 3069 goto fail; 3070 } 3071 mp = rxbuf->m_pack; 3072 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3073 /* Get the memory mapping */ 3074 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3075 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3076 if (error != 0) 3077 goto fail; 3078 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3079 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 3080 /* Update descriptor */ 3081 rxr->rx_base[j].read.pkt_addr = 3082 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3083 } 3084 3085 3086 /* Setup our descriptor indices */ 3087 rxr->next_to_check = 0; 3088 rxr->next_to_refresh = 0; 3089 rxr->lro_enabled = FALSE; 3090 rxr->rx_split_packets.ev_count = 0; 3091 rxr->rx_bytes.ev_count = 0; 3092 rxr->discard = FALSE; 3093 3094 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3095 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3096 3097 #ifdef LRO 3098 /* 3099 ** Now set up the LRO interface: 3100 */ 3101 if (ifp->if_capenable & IFCAP_LRO) { 3102 device_t dev = adapter->dev; 3103 int err = tcp_lro_init(lro); 3104 if (err) { 3105 device_printf(dev, "LRO Initialization failed!\n"); 3106 goto fail; 3107 } 3108 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 3109 rxr->lro_enabled = TRUE; 3110 lro->ifp = adapter->ifp; 3111 } 3112 #endif /* LRO */ 3113 3114 IXV_RX_UNLOCK(rxr); 3115 return (0); 3116 3117 fail: 3118 ixv_free_receive_ring(rxr); 3119 IXV_RX_UNLOCK(rxr); 3120 return (error); 3121 } 3122 3123 /********************************************************************* 3124 * 3125 * Initialize all receive rings. 3126 * 3127 **********************************************************************/ 3128 static int 3129 ixv_setup_receive_structures(struct adapter *adapter) 3130 { 3131 struct rx_ring *rxr = adapter->rx_rings; 3132 int j; 3133 3134 for (j = 0; j < adapter->num_queues; j++, rxr++) 3135 if (ixv_setup_receive_ring(rxr)) 3136 goto fail; 3137 3138 return (0); 3139 fail: 3140 /* 3141 * Free RX buffers allocated so far, we will only handle 3142 * the rings that completed, the failing case will have 3143 * cleaned up for itself. 'j' failed, so its the terminus. 3144 */ 3145 for (int i = 0; i < j; ++i) { 3146 rxr = &adapter->rx_rings[i]; 3147 ixv_free_receive_ring(rxr); 3148 } 3149 3150 return (ENOBUFS); 3151 } 3152 3153 /********************************************************************* 3154 * 3155 * Setup receive registers and features. 3156 * 3157 **********************************************************************/ 3158 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3159 3160 static void 3161 ixv_initialize_receive_units(struct adapter *adapter) 3162 { 3163 int i; 3164 struct rx_ring *rxr = adapter->rx_rings; 3165 struct ixgbe_hw *hw = &adapter->hw; 3166 struct ifnet *ifp = adapter->ifp; 3167 u32 bufsz, fctrl, rxcsum, hlreg; 3168 3169 3170 /* Enable broadcasts */ 3171 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3172 fctrl |= IXGBE_FCTRL_BAM; 3173 fctrl |= IXGBE_FCTRL_DPF; 3174 fctrl |= IXGBE_FCTRL_PMCF; 3175 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3176 3177 /* Set for Jumbo Frames? */ 3178 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3179 if (ifp->if_mtu > ETHERMTU) { 3180 hlreg |= IXGBE_HLREG0_JUMBOEN; 3181 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3182 } else { 3183 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3184 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3185 } 3186 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3187 3188 for (i = 0; i < adapter->num_queues; i++, rxr++) { 3189 u64 rdba = rxr->rxdma.dma_paddr; 3190 u32 reg, rxdctl; 3191 3192 /* Do the queue enabling first */ 3193 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3194 rxdctl |= IXGBE_RXDCTL_ENABLE; 3195 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3196 for (int k = 0; k < 10; k++) { 3197 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3198 IXGBE_RXDCTL_ENABLE) 3199 break; 3200 else 3201 msec_delay(1); 3202 } 3203 wmb(); 3204 3205 /* Setup the Base and Length of the Rx Descriptor Ring */ 3206 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3207 (rdba & 0x00000000ffffffffULL)); 3208 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3209 (rdba >> 32)); 3210 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3211 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3212 3213 /* Set up the SRRCTL register */ 3214 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3215 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3216 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3217 reg |= bufsz; 3218 if (rxr->hdr_split) { 3219 /* Use a standard mbuf for the header */ 3220 reg |= ((IXV_RX_HDR << 3221 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3222 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3223 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3224 } else 3225 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3226 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3227 3228 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3229 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3230 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3231 adapter->num_rx_desc - 1); 3232 } 3233 3234 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3235 3236 if (ifp->if_capenable & IFCAP_RXCSUM) 3237 rxcsum |= IXGBE_RXCSUM_PCSD; 3238 3239 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3240 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3241 3242 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3243 3244 return; 3245 } 3246 3247 /********************************************************************* 3248 * 3249 * Free all receive rings. 3250 * 3251 **********************************************************************/ 3252 static void 3253 ixv_free_receive_structures(struct adapter *adapter) 3254 { 3255 struct rx_ring *rxr = adapter->rx_rings; 3256 3257 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3258 #ifdef LRO 3259 struct lro_ctrl *lro = &rxr->lro; 3260 #endif /* LRO */ 3261 ixv_free_receive_buffers(rxr); 3262 #ifdef LRO 3263 /* Free LRO memory */ 3264 tcp_lro_free(lro); 3265 #endif /* LRO */ 3266 /* Free the ring memory as well */ 3267 ixv_dma_free(adapter, &rxr->rxdma); 3268 IXV_RX_LOCK_DESTROY(rxr); 3269 } 3270 3271 free(adapter->rx_rings, M_DEVBUF); 3272 } 3273 3274 3275 /********************************************************************* 3276 * 3277 * Free receive ring data structures 3278 * 3279 **********************************************************************/ 3280 static void 3281 ixv_free_receive_buffers(struct rx_ring *rxr) 3282 { 3283 struct adapter *adapter = rxr->adapter; 3284 struct ixv_rx_buf *rxbuf; 3285 3286 INIT_DEBUGOUT("free_receive_structures: begin"); 3287 3288 /* Cleanup any existing buffers */ 3289 if (rxr->rx_buffers != NULL) { 3290 for (int i = 0; i < adapter->num_rx_desc; i++) { 3291 rxbuf = &rxr->rx_buffers[i]; 3292 if (rxbuf->m_head != NULL) { 3293 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 3294 BUS_DMASYNC_POSTREAD); 3295 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 3296 rxbuf->m_head->m_flags |= M_PKTHDR; 3297 m_freem(rxbuf->m_head); 3298 } 3299 if (rxbuf->m_pack != NULL) { 3300 /* XXX not ixgbe_* ? */ 3301 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3302 0, rxbuf->m_pack->m_pkthdr.len, 3303 BUS_DMASYNC_POSTREAD); 3304 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 3305 rxbuf->m_pack->m_flags |= M_PKTHDR; 3306 m_freem(rxbuf->m_pack); 3307 } 3308 rxbuf->m_head = NULL; 3309 rxbuf->m_pack = NULL; 3310 if (rxbuf->hmap != NULL) { 3311 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 3312 rxbuf->hmap = NULL; 3313 } 3314 if (rxbuf->pmap != NULL) { 3315 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3316 rxbuf->pmap = NULL; 3317 } 3318 } 3319 if (rxr->rx_buffers != NULL) { 3320 free(rxr->rx_buffers, M_DEVBUF); 3321 rxr->rx_buffers = NULL; 3322 } 3323 } 3324 3325 if (rxr->htag != NULL) { 3326 ixgbe_dma_tag_destroy(rxr->htag); 3327 rxr->htag = NULL; 3328 } 3329 if (rxr->ptag != NULL) { 3330 ixgbe_dma_tag_destroy(rxr->ptag); 3331 rxr->ptag = NULL; 3332 } 3333 3334 return; 3335 } 3336 3337 static __inline void 3338 ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3339 { 3340 int s; 3341 3342 #ifdef LRO 3343 struct adapter *adapter = ifp->if_softc; 3344 struct ethercom *ec = &adapter->osdep.ec; 3345 3346 /* 3347 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3348 * should be computed by hardware. Also it should not have VLAN tag in 3349 * ethernet header. 3350 */ 3351 if (rxr->lro_enabled && 3352 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 3353 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3354 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3355 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3356 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3357 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3358 /* 3359 * Send to the stack if: 3360 ** - LRO not enabled, or 3361 ** - no LRO resources, or 3362 ** - lro enqueue fails 3363 */ 3364 if (rxr->lro.lro_cnt != 0) 3365 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3366 return; 3367 } 3368 #endif /* LRO */ 3369 3370 IXV_RX_UNLOCK(rxr); 3371 3372 s = splnet(); 3373 /* Pass this up to any BPF listeners. */ 3374 bpf_mtap(ifp, m); 3375 if_percpuq_enqueue(ifp->if_percpuq, m); 3376 splx(s); 3377 3378 IXV_RX_LOCK(rxr); 3379 } 3380 3381 static __inline void 3382 ixv_rx_discard(struct rx_ring *rxr, int i) 3383 { 3384 struct ixv_rx_buf *rbuf; 3385 3386 rbuf = &rxr->rx_buffers[i]; 3387 if (rbuf->fmp != NULL) {/* Partial chain ? */ 3388 rbuf->fmp->m_flags |= M_PKTHDR; 3389 m_freem(rbuf->fmp); 3390 rbuf->fmp = NULL; 3391 } 3392 3393 /* 3394 ** With advanced descriptors the writeback 3395 ** clobbers the buffer addrs, so its easier 3396 ** to just free the existing mbufs and take 3397 ** the normal refresh path to get new buffers 3398 ** and mapping. 3399 */ 3400 if (rbuf->m_head) { 3401 m_free(rbuf->m_head); 3402 rbuf->m_head = NULL; 3403 } 3404 3405 if (rbuf->m_pack) { 3406 m_free(rbuf->m_pack); 3407 rbuf->m_pack = NULL; 3408 } 3409 3410 return; 3411 } 3412 3413 3414 /********************************************************************* 3415 * 3416 * This routine executes in interrupt context. It replenishes 3417 * the mbufs in the descriptor and sends data which has been 3418 * dma'ed into host memory to upper layer. 3419 * 3420 * We loop at most count times if count is > 0, or until done if 3421 * count < 0. 3422 * 3423 * Return TRUE for more work, FALSE for all clean. 3424 *********************************************************************/ 3425 static bool 3426 ixv_rxeof(struct ix_queue *que, int count) 3427 { 3428 struct adapter *adapter = que->adapter; 3429 struct rx_ring *rxr = que->rxr; 3430 struct ifnet *ifp = adapter->ifp; 3431 #ifdef LRO 3432 struct lro_ctrl *lro = &rxr->lro; 3433 struct lro_entry *queued; 3434 #endif /* LRO */ 3435 int i, nextp, processed = 0; 3436 u32 staterr = 0; 3437 union ixgbe_adv_rx_desc *cur; 3438 struct ixv_rx_buf *rbuf, *nbuf; 3439 3440 IXV_RX_LOCK(rxr); 3441 3442 for (i = rxr->next_to_check; count != 0;) { 3443 struct mbuf *sendmp, *mh, *mp; 3444 u32 ptype; 3445 u16 hlen, plen, hdr, vtag; 3446 bool eop; 3447 3448 /* Sync the ring. */ 3449 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3450 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3451 3452 cur = &rxr->rx_base[i]; 3453 staterr = le32toh(cur->wb.upper.status_error); 3454 3455 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3456 break; 3457 if ((ifp->if_flags & IFF_RUNNING) == 0) 3458 break; 3459 3460 count--; 3461 sendmp = NULL; 3462 nbuf = NULL; 3463 cur->wb.upper.status_error = 0; 3464 rbuf = &rxr->rx_buffers[i]; 3465 mh = rbuf->m_head; 3466 mp = rbuf->m_pack; 3467 3468 plen = le16toh(cur->wb.upper.length); 3469 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3470 IXGBE_RXDADV_PKTTYPE_MASK; 3471 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3472 vtag = le16toh(cur->wb.upper.vlan); 3473 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3474 3475 /* Make sure all parts of a bad packet are discarded */ 3476 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3477 (rxr->discard)) { 3478 ifp->if_ierrors++; 3479 rxr->rx_discarded.ev_count++; 3480 if (!eop) 3481 rxr->discard = TRUE; 3482 else 3483 rxr->discard = FALSE; 3484 ixv_rx_discard(rxr, i); 3485 goto next_desc; 3486 } 3487 3488 if (!eop) { 3489 nextp = i + 1; 3490 if (nextp == adapter->num_rx_desc) 3491 nextp = 0; 3492 nbuf = &rxr->rx_buffers[nextp]; 3493 prefetch(nbuf); 3494 } 3495 /* 3496 ** The header mbuf is ONLY used when header 3497 ** split is enabled, otherwise we get normal 3498 ** behavior, ie, both header and payload 3499 ** are DMA'd into the payload buffer. 3500 ** 3501 ** Rather than using the fmp/lmp global pointers 3502 ** we now keep the head of a packet chain in the 3503 ** buffer struct and pass this along from one 3504 ** descriptor to the next, until we get EOP. 3505 */ 3506 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3507 /* This must be an initial descriptor */ 3508 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3509 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3510 if (hlen > IXV_RX_HDR) 3511 hlen = IXV_RX_HDR; 3512 mh->m_len = hlen; 3513 mh->m_flags |= M_PKTHDR; 3514 mh->m_next = NULL; 3515 mh->m_pkthdr.len = mh->m_len; 3516 /* Null buf pointer so it is refreshed */ 3517 rbuf->m_head = NULL; 3518 /* 3519 ** Check the payload length, this 3520 ** could be zero if its a small 3521 ** packet. 3522 */ 3523 if (plen > 0) { 3524 mp->m_len = plen; 3525 mp->m_next = NULL; 3526 mp->m_flags &= ~M_PKTHDR; 3527 mh->m_next = mp; 3528 mh->m_pkthdr.len += mp->m_len; 3529 /* Null buf pointer so it is refreshed */ 3530 rbuf->m_pack = NULL; 3531 rxr->rx_split_packets.ev_count++; 3532 } 3533 /* 3534 ** Now create the forward 3535 ** chain so when complete 3536 ** we wont have to. 3537 */ 3538 if (eop == 0) { 3539 /* stash the chain head */ 3540 nbuf->fmp = mh; 3541 /* Make forward chain */ 3542 if (plen) 3543 mp->m_next = nbuf->m_pack; 3544 else 3545 mh->m_next = nbuf->m_pack; 3546 } else { 3547 /* Singlet, prepare to send */ 3548 sendmp = mh; 3549 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3550 (staterr & IXGBE_RXD_STAT_VP)) { 3551 VLAN_INPUT_TAG(ifp, sendmp, vtag, 3552 printf("%s: could not apply VLAN " 3553 "tag", __func__)); 3554 } 3555 } 3556 } else { 3557 /* 3558 ** Either no header split, or a 3559 ** secondary piece of a fragmented 3560 ** split packet. 3561 */ 3562 mp->m_len = plen; 3563 /* 3564 ** See if there is a stored head 3565 ** that determines what we are 3566 */ 3567 sendmp = rbuf->fmp; 3568 rbuf->m_pack = rbuf->fmp = NULL; 3569 3570 if (sendmp != NULL) /* secondary frag */ 3571 sendmp->m_pkthdr.len += mp->m_len; 3572 else { 3573 /* first desc of a non-ps chain */ 3574 sendmp = mp; 3575 sendmp->m_flags |= M_PKTHDR; 3576 sendmp->m_pkthdr.len = mp->m_len; 3577 if (staterr & IXGBE_RXD_STAT_VP) { 3578 /* XXX Do something reasonable on 3579 * error. 3580 */ 3581 VLAN_INPUT_TAG(ifp, sendmp, vtag, 3582 printf("%s: could not apply VLAN " 3583 "tag", __func__)); 3584 } 3585 } 3586 /* Pass the head pointer on */ 3587 if (eop == 0) { 3588 nbuf->fmp = sendmp; 3589 sendmp = NULL; 3590 mp->m_next = nbuf->m_pack; 3591 } 3592 } 3593 ++processed; 3594 /* Sending this frame? */ 3595 if (eop) { 3596 m_set_rcvif(sendmp, ifp); 3597 ifp->if_ipackets++; 3598 rxr->rx_packets.ev_count++; 3599 /* capture data for AIM */ 3600 rxr->bytes += sendmp->m_pkthdr.len; 3601 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 3602 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 3603 ixv_rx_checksum(staterr, sendmp, ptype, 3604 &adapter->stats); 3605 } 3606 #if __FreeBSD_version >= 800000 3607 sendmp->m_pkthdr.flowid = que->msix; 3608 sendmp->m_flags |= M_FLOWID; 3609 #endif 3610 } 3611 next_desc: 3612 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3613 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3614 3615 /* Advance our pointers to the next descriptor. */ 3616 if (++i == adapter->num_rx_desc) 3617 i = 0; 3618 3619 /* Now send to the stack or do LRO */ 3620 if (sendmp != NULL) 3621 ixv_rx_input(rxr, ifp, sendmp, ptype); 3622 3623 /* Every 8 descriptors we go to refresh mbufs */ 3624 if (processed == 8) { 3625 ixv_refresh_mbufs(rxr, i); 3626 processed = 0; 3627 } 3628 } 3629 3630 /* Refresh any remaining buf structs */ 3631 if (ixv_rx_unrefreshed(rxr)) 3632 ixv_refresh_mbufs(rxr, i); 3633 3634 rxr->next_to_check = i; 3635 3636 #ifdef LRO 3637 /* 3638 * Flush any outstanding LRO work 3639 */ 3640 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3641 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3642 tcp_lro_flush(lro, queued); 3643 } 3644 #endif /* LRO */ 3645 3646 IXV_RX_UNLOCK(rxr); 3647 3648 /* 3649 ** We still have cleaning to do? 3650 ** Schedule another interrupt if so. 3651 */ 3652 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3653 ixv_rearm_queues(adapter, (u64)(1ULL << que->msix)); 3654 return true; 3655 } 3656 3657 return false; 3658 } 3659 3660 3661 /********************************************************************* 3662 * 3663 * Verify that the hardware indicated that the checksum is valid. 3664 * Inform the stack about the status of checksum so that stack 3665 * doesn't spend time verifying the checksum. 3666 * 3667 *********************************************************************/ 3668 static void 3669 ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 3670 struct ixgbevf_hw_stats *stats) 3671 { 3672 u16 status = (u16) staterr; 3673 u8 errors = (u8) (staterr >> 24); 3674 #if 0 3675 bool sctp = FALSE; 3676 3677 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3678 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3679 sctp = TRUE; 3680 #endif 3681 if (status & IXGBE_RXD_STAT_IPCS) { 3682 stats->ipcs.ev_count++; 3683 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3684 /* IP Checksum Good */ 3685 mp->m_pkthdr.csum_flags |= M_CSUM_IPv4; 3686 3687 } else { 3688 stats->ipcs_bad.ev_count++; 3689 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 3690 } 3691 } 3692 if (status & IXGBE_RXD_STAT_L4CS) { 3693 stats->l4cs.ev_count++; 3694 int type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 3695 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3696 mp->m_pkthdr.csum_flags |= type; 3697 } else { 3698 stats->l4cs_bad.ev_count++; 3699 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 3700 } 3701 } 3702 return; 3703 } 3704 3705 static void 3706 ixv_setup_vlan_support(struct adapter *adapter) 3707 { 3708 struct ixgbe_hw *hw = &adapter->hw; 3709 u32 ctrl, vid, vfta, retry; 3710 3711 3712 /* 3713 ** We get here thru init_locked, meaning 3714 ** a soft reset, this has already cleared 3715 ** the VFTA and other state, so if there 3716 ** have been no vlan's registered do nothing. 3717 */ 3718 if (adapter->num_vlans == 0) 3719 return; 3720 3721 /* Enable the queues */ 3722 for (int i = 0; i < adapter->num_queues; i++) { 3723 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3724 ctrl |= IXGBE_RXDCTL_VME; 3725 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3726 } 3727 3728 /* 3729 ** A soft reset zero's out the VFTA, so 3730 ** we need to repopulate it now. 3731 */ 3732 for (int i = 0; i < VFTA_SIZE; i++) { 3733 if (ixv_shadow_vfta[i] == 0) 3734 continue; 3735 vfta = ixv_shadow_vfta[i]; 3736 /* 3737 ** Reconstruct the vlan id's 3738 ** based on the bits set in each 3739 ** of the array ints. 3740 */ 3741 for ( int j = 0; j < 32; j++) { 3742 retry = 0; 3743 if ((vfta & (1 << j)) == 0) 3744 continue; 3745 vid = (i * 32) + j; 3746 /* Call the shared code mailbox routine */ 3747 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3748 if (++retry > 5) 3749 break; 3750 } 3751 } 3752 } 3753 } 3754 3755 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 3756 /* 3757 ** This routine is run via an vlan config EVENT, 3758 ** it enables us to use the HW Filter table since 3759 ** we can get the vlan id. This just creates the 3760 ** entry in the soft version of the VFTA, init will 3761 ** repopulate the real table. 3762 */ 3763 static void 3764 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3765 { 3766 struct adapter *adapter = ifp->if_softc; 3767 u16 index, bit; 3768 3769 if (ifp->if_softc != arg) /* Not our event */ 3770 return; 3771 3772 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3773 return; 3774 3775 IXV_CORE_LOCK(adapter); 3776 index = (vtag >> 5) & 0x7F; 3777 bit = vtag & 0x1F; 3778 ixv_shadow_vfta[index] |= (1 << bit); 3779 /* Re-init to load the changes */ 3780 ixv_init_locked(adapter); 3781 IXV_CORE_UNLOCK(adapter); 3782 } 3783 3784 /* 3785 ** This routine is run via an vlan 3786 ** unconfig EVENT, remove our entry 3787 ** in the soft vfta. 3788 */ 3789 static void 3790 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3791 { 3792 struct adapter *adapter = ifp->if_softc; 3793 u16 index, bit; 3794 3795 if (ifp->if_softc != arg) 3796 return; 3797 3798 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3799 return; 3800 3801 IXV_CORE_LOCK(adapter); 3802 index = (vtag >> 5) & 0x7F; 3803 bit = vtag & 0x1F; 3804 ixv_shadow_vfta[index] &= ~(1 << bit); 3805 /* Re-init to load the changes */ 3806 ixv_init_locked(adapter); 3807 IXV_CORE_UNLOCK(adapter); 3808 } 3809 #endif 3810 3811 static void 3812 ixv_enable_intr(struct adapter *adapter) 3813 { 3814 struct ixgbe_hw *hw = &adapter->hw; 3815 struct ix_queue *que = adapter->queues; 3816 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3817 3818 3819 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3820 3821 mask = IXGBE_EIMS_ENABLE_MASK; 3822 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3823 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3824 3825 for (int i = 0; i < adapter->num_queues; i++, que++) 3826 ixv_enable_queue(adapter, que->msix); 3827 3828 IXGBE_WRITE_FLUSH(hw); 3829 3830 return; 3831 } 3832 3833 static void 3834 ixv_disable_intr(struct adapter *adapter) 3835 { 3836 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3837 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3838 IXGBE_WRITE_FLUSH(&adapter->hw); 3839 return; 3840 } 3841 3842 /* 3843 ** Setup the correct IVAR register for a particular MSIX interrupt 3844 ** - entry is the register array entry 3845 ** - vector is the MSIX vector for this queue 3846 ** - type is RX/TX/MISC 3847 */ 3848 static void 3849 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3850 { 3851 struct ixgbe_hw *hw = &adapter->hw; 3852 u32 ivar, index; 3853 3854 vector |= IXGBE_IVAR_ALLOC_VAL; 3855 3856 if (type == -1) { /* MISC IVAR */ 3857 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3858 ivar &= ~0xFF; 3859 ivar |= vector; 3860 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3861 } else { /* RX/TX IVARS */ 3862 index = (16 * (entry & 1)) + (8 * type); 3863 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3864 ivar &= ~(0xFF << index); 3865 ivar |= (vector << index); 3866 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3867 } 3868 } 3869 3870 static void 3871 ixv_configure_ivars(struct adapter *adapter) 3872 { 3873 struct ix_queue *que = adapter->queues; 3874 3875 for (int i = 0; i < adapter->num_queues; i++, que++) { 3876 /* First the RX queue entry */ 3877 ixv_set_ivar(adapter, i, que->msix, 0); 3878 /* ... and the TX */ 3879 ixv_set_ivar(adapter, i, que->msix, 1); 3880 /* Set an initial value in EITR */ 3881 IXGBE_WRITE_REG(&adapter->hw, 3882 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3883 } 3884 3885 /* For the Link interrupt */ 3886 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3887 } 3888 3889 3890 /* 3891 ** Tasklet handler for MSIX MBX interrupts 3892 ** - do outside interrupt since it might sleep 3893 */ 3894 static void 3895 ixv_handle_mbx(void *context) 3896 { 3897 struct adapter *adapter = context; 3898 3899 ixgbe_check_link(&adapter->hw, 3900 &adapter->link_speed, &adapter->link_up, 0); 3901 ixv_update_link_status(adapter); 3902 } 3903 3904 /* 3905 ** The VF stats registers never have a truely virgin 3906 ** starting point, so this routine tries to make an 3907 ** artificial one, marking ground zero on attach as 3908 ** it were. 3909 */ 3910 static void 3911 ixv_save_stats(struct adapter *adapter) 3912 { 3913 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3914 adapter->stats.saved_reset_vfgprc += 3915 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3916 adapter->stats.saved_reset_vfgptc += 3917 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3918 adapter->stats.saved_reset_vfgorc += 3919 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3920 adapter->stats.saved_reset_vfgotc += 3921 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3922 adapter->stats.saved_reset_vfmprc += 3923 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3924 } 3925 } 3926 3927 static void 3928 ixv_init_stats(struct adapter *adapter) 3929 { 3930 struct ixgbe_hw *hw = &adapter->hw; 3931 3932 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3933 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3934 adapter->stats.last_vfgorc |= 3935 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3936 3937 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3938 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3939 adapter->stats.last_vfgotc |= 3940 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3941 3942 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3943 3944 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3945 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3946 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3947 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3948 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3949 } 3950 3951 #define UPDATE_STAT_32(reg, last, count) \ 3952 { \ 3953 u32 current = IXGBE_READ_REG(hw, reg); \ 3954 if (current < last) \ 3955 count += 0x100000000LL; \ 3956 last = current; \ 3957 count &= 0xFFFFFFFF00000000LL; \ 3958 count |= current; \ 3959 } 3960 3961 #define UPDATE_STAT_36(lsb, msb, last, count) \ 3962 { \ 3963 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3964 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3965 u64 current = ((cur_msb << 32) | cur_lsb); \ 3966 if (current < last) \ 3967 count += 0x1000000000LL; \ 3968 last = current; \ 3969 count &= 0xFFFFFFF000000000LL; \ 3970 count |= current; \ 3971 } 3972 3973 /* 3974 ** ixv_update_stats - Update the board statistics counters. 3975 */ 3976 void 3977 ixv_update_stats(struct adapter *adapter) 3978 { 3979 struct ixgbe_hw *hw = &adapter->hw; 3980 3981 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3982 adapter->stats.vfgprc); 3983 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3984 adapter->stats.vfgptc); 3985 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3986 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3987 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3988 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3989 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3990 adapter->stats.vfmprc); 3991 } 3992 3993 /********************************************************************** 3994 * 3995 * This routine is called only when ixgbe_display_debug_stats is enabled. 3996 * This routine provides a way to take a look at important statistics 3997 * maintained by the driver and hardware. 3998 * 3999 **********************************************************************/ 4000 static void 4001 ixv_print_hw_stats(struct adapter * adapter) 4002 { 4003 device_t dev = adapter->dev; 4004 4005 device_printf(dev,"Std Mbuf Failed = %"PRIu64"\n", 4006 adapter->mbuf_defrag_failed.ev_count); 4007 device_printf(dev,"Driver dropped packets = %"PRIu64"\n", 4008 adapter->dropped_pkts.ev_count); 4009 device_printf(dev, "watchdog timeouts = %"PRIu64"\n", 4010 adapter->watchdog_events.ev_count); 4011 4012 device_printf(dev,"Good Packets Rcvd = %lld\n", 4013 (long long)adapter->stats.vfgprc); 4014 device_printf(dev,"Good Packets Xmtd = %lld\n", 4015 (long long)adapter->stats.vfgptc); 4016 device_printf(dev,"TSO Transmissions = %"PRIu64"\n", 4017 adapter->tso_tx.ev_count); 4018 4019 } 4020 4021 /********************************************************************** 4022 * 4023 * This routine is called only when em_display_debug_stats is enabled. 4024 * This routine provides a way to take a look at important statistics 4025 * maintained by the driver and hardware. 4026 * 4027 **********************************************************************/ 4028 static void 4029 ixv_print_debug_info(struct adapter *adapter) 4030 { 4031 device_t dev = adapter->dev; 4032 struct ixgbe_hw *hw = &adapter->hw; 4033 struct ix_queue *que = adapter->queues; 4034 struct rx_ring *rxr; 4035 struct tx_ring *txr; 4036 #ifdef LRO 4037 struct lro_ctrl *lro; 4038 #endif /* LRO */ 4039 4040 device_printf(dev,"Error Byte Count = %u \n", 4041 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 4042 4043 for (int i = 0; i < adapter->num_queues; i++, que++) { 4044 txr = que->txr; 4045 rxr = que->rxr; 4046 #ifdef LRO 4047 lro = &rxr->lro; 4048 #endif /* LRO */ 4049 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 4050 que->msix, (long)que->irqs); 4051 device_printf(dev,"RX(%d) Packets Received: %lld\n", 4052 rxr->me, (long long)rxr->rx_packets.ev_count); 4053 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 4054 rxr->me, (long long)rxr->rx_split_packets.ev_count); 4055 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 4056 rxr->me, (long)rxr->rx_bytes.ev_count); 4057 #ifdef LRO 4058 device_printf(dev,"RX(%d) LRO Queued= %d\n", 4059 rxr->me, lro->lro_queued); 4060 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 4061 rxr->me, lro->lro_flushed); 4062 #endif /* LRO */ 4063 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 4064 txr->me, (long)txr->total_packets.ev_count); 4065 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 4066 txr->me, (long)txr->no_desc_avail.ev_count); 4067 } 4068 4069 device_printf(dev,"MBX IRQ Handled: %lu\n", 4070 (long)adapter->mbx_irq.ev_count); 4071 return; 4072 } 4073 4074 static int 4075 ixv_sysctl_stats(SYSCTLFN_ARGS) 4076 { 4077 struct sysctlnode node; 4078 int error; 4079 int result; 4080 struct adapter *adapter; 4081 4082 node = *rnode; 4083 adapter = (struct adapter *)node.sysctl_data; 4084 node.sysctl_data = &result; 4085 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4086 if (error != 0) 4087 return error; 4088 4089 if (result == 1) 4090 ixv_print_hw_stats(adapter); 4091 4092 return 0; 4093 } 4094 4095 static int 4096 ixv_sysctl_debug(SYSCTLFN_ARGS) 4097 { 4098 struct sysctlnode node; 4099 int error, result; 4100 struct adapter *adapter; 4101 4102 node = *rnode; 4103 adapter = (struct adapter *)node.sysctl_data; 4104 node.sysctl_data = &result; 4105 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4106 4107 if (error) 4108 return error; 4109 4110 if (result == 1) 4111 ixv_print_debug_info(adapter); 4112 4113 return 0; 4114 } 4115 4116 /* 4117 ** Set flow control using sysctl: 4118 ** Flow control values: 4119 ** 0 - off 4120 ** 1 - rx pause 4121 ** 2 - tx pause 4122 ** 3 - full 4123 */ 4124 static int 4125 ixv_set_flowcntl(SYSCTLFN_ARGS) 4126 { 4127 struct sysctlnode node; 4128 int error; 4129 struct adapter *adapter; 4130 4131 node = *rnode; 4132 adapter = (struct adapter *)node.sysctl_data; 4133 node.sysctl_data = &ixv_flow_control; 4134 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4135 4136 if (error) 4137 return (error); 4138 4139 switch (ixv_flow_control) { 4140 case ixgbe_fc_rx_pause: 4141 case ixgbe_fc_tx_pause: 4142 case ixgbe_fc_full: 4143 adapter->hw.fc.requested_mode = ixv_flow_control; 4144 break; 4145 case ixgbe_fc_none: 4146 default: 4147 adapter->hw.fc.requested_mode = ixgbe_fc_none; 4148 } 4149 4150 ixgbe_fc_enable(&adapter->hw); 4151 return error; 4152 } 4153 4154 const struct sysctlnode * 4155 ixv_sysctl_instance(struct adapter *adapter) 4156 { 4157 const char *dvname; 4158 struct sysctllog **log; 4159 int rc; 4160 const struct sysctlnode *rnode; 4161 4162 log = &adapter->sysctllog; 4163 dvname = device_xname(adapter->dev); 4164 4165 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 4166 0, CTLTYPE_NODE, dvname, 4167 SYSCTL_DESCR("ixv information and settings"), 4168 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 4169 goto err; 4170 4171 return rnode; 4172 err: 4173 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 4174 return NULL; 4175 } 4176 4177 static void 4178 ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 4179 const char *description, int *limit, int value) 4180 { 4181 const struct sysctlnode *rnode, *cnode; 4182 struct sysctllog **log = &adapter->sysctllog; 4183 4184 *limit = value; 4185 4186 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) 4187 aprint_error_dev(adapter->dev, 4188 "could not create sysctl root\n"); 4189 else if (sysctl_createv(log, 0, &rnode, &cnode, 4190 CTLFLAG_READWRITE, 4191 CTLTYPE_INT, 4192 name, SYSCTL_DESCR(description), 4193 NULL, 0, limit, 0, 4194 CTL_CREATE, CTL_EOL) != 0) { 4195 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 4196 __func__); 4197 } 4198 } 4199 4200