1 /* 2 * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>. All rights reserved. 3 * 4 * Copyright (c) 2001-2014, Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * 34 * Copyright (c) 2005 The DragonFly Project. All rights reserved. 35 * 36 * This code is derived from software contributed to The DragonFly Project 37 * by Matthew Dillon <dillon@backplane.com> 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in 47 * the documentation and/or other materials provided with the 48 * distribution. 49 * 3. Neither the name of The DragonFly Project nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific, prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 55 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 56 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 57 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 58 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 59 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 60 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 61 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 62 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 63 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 * 66 */ 67 /* 68 * SERIALIZATION API RULES: 69 * 70 * - We must call lwkt_serialize_handler_enable() prior to enabling the 71 * hardware interrupt and lwkt_serialize_handler_disable() after disabling 72 * the hardware interrupt in order to avoid handler execution races from 73 * scheduled interrupt threads. 74 */ 75 76 #include "opt_ifpoll.h" 77 78 #include <sys/param.h> 79 #include <sys/bus.h> 80 #include <sys/endian.h> 81 #include <sys/interrupt.h> 82 #include <sys/kernel.h> 83 #include <sys/ktr.h> 84 #include <sys/malloc.h> 85 #include <sys/mbuf.h> 86 #include <sys/proc.h> 87 #include <sys/rman.h> 88 #include <sys/serialize.h> 89 #include <sys/socket.h> 90 #include <sys/sockio.h> 91 #include <sys/sysctl.h> 92 #include <sys/systm.h> 93 94 #include <net/bpf.h> 95 #include <net/ethernet.h> 96 #include <net/if.h> 97 #include <net/if_arp.h> 98 #include <net/if_dl.h> 99 #include <net/if_media.h> 100 #include <net/if_poll.h> 101 #include <net/ifq_var.h> 102 #include <net/vlan/if_vlan_var.h> 103 #include <net/vlan/if_vlan_ether.h> 104 105 #include <netinet/ip.h> 106 #include <netinet/tcp.h> 107 #include <netinet/udp.h> 108 109 #include <bus/pci/pcivar.h> 110 #include <bus/pci/pcireg.h> 111 112 #include <dev/netif/ig_hal/e1000_api.h> 113 #include <dev/netif/ig_hal/e1000_82571.h> 114 #include <dev/netif/ig_hal/e1000_dragonfly.h> 115 #include <dev/netif/em/if_em.h> 116 117 #define DEBUG_HW 0 118 119 #define EM_NAME "Intel(R) PRO/1000 Network Connection " 120 #define EM_VER " 7.4.2" 121 122 #define _EM_DEVICE(id, ret) \ 123 { EM_VENDOR_ID, E1000_DEV_ID_##id, ret, EM_NAME #id EM_VER } 124 #define EM_EMX_DEVICE(id) _EM_DEVICE(id, -100) 125 #define EM_DEVICE(id) _EM_DEVICE(id, 0) 126 #define EM_DEVICE_NULL { 0, 0, 0, NULL } 127 128 static const struct em_vendor_info em_vendor_info_array[] = { 129 EM_DEVICE(82540EM), 130 EM_DEVICE(82540EM_LOM), 131 EM_DEVICE(82540EP), 132 EM_DEVICE(82540EP_LOM), 133 EM_DEVICE(82540EP_LP), 134 135 EM_DEVICE(82541EI), 136 EM_DEVICE(82541ER), 137 EM_DEVICE(82541ER_LOM), 138 EM_DEVICE(82541EI_MOBILE), 139 EM_DEVICE(82541GI), 140 EM_DEVICE(82541GI_LF), 141 EM_DEVICE(82541GI_MOBILE), 142 143 EM_DEVICE(82542), 144 145 EM_DEVICE(82543GC_FIBER), 146 EM_DEVICE(82543GC_COPPER), 147 148 EM_DEVICE(82544EI_COPPER), 149 EM_DEVICE(82544EI_FIBER), 150 EM_DEVICE(82544GC_COPPER), 151 EM_DEVICE(82544GC_LOM), 152 153 EM_DEVICE(82545EM_COPPER), 154 EM_DEVICE(82545EM_FIBER), 155 EM_DEVICE(82545GM_COPPER), 156 EM_DEVICE(82545GM_FIBER), 157 EM_DEVICE(82545GM_SERDES), 158 159 EM_DEVICE(82546EB_COPPER), 160 EM_DEVICE(82546EB_FIBER), 161 EM_DEVICE(82546EB_QUAD_COPPER), 162 EM_DEVICE(82546GB_COPPER), 163 EM_DEVICE(82546GB_FIBER), 164 EM_DEVICE(82546GB_SERDES), 165 EM_DEVICE(82546GB_PCIE), 166 EM_DEVICE(82546GB_QUAD_COPPER), 167 EM_DEVICE(82546GB_QUAD_COPPER_KSP3), 168 169 EM_DEVICE(82547EI), 170 EM_DEVICE(82547EI_MOBILE), 171 EM_DEVICE(82547GI), 172 173 EM_EMX_DEVICE(82571EB_COPPER), 174 EM_EMX_DEVICE(82571EB_FIBER), 175 EM_EMX_DEVICE(82571EB_SERDES), 176 EM_EMX_DEVICE(82571EB_SERDES_DUAL), 177 EM_EMX_DEVICE(82571EB_SERDES_QUAD), 178 EM_EMX_DEVICE(82571EB_QUAD_COPPER), 179 EM_EMX_DEVICE(82571EB_QUAD_COPPER_BP), 180 EM_EMX_DEVICE(82571EB_QUAD_COPPER_LP), 181 EM_EMX_DEVICE(82571EB_QUAD_FIBER), 182 EM_EMX_DEVICE(82571PT_QUAD_COPPER), 183 184 EM_EMX_DEVICE(82572EI_COPPER), 185 EM_EMX_DEVICE(82572EI_FIBER), 186 EM_EMX_DEVICE(82572EI_SERDES), 187 EM_EMX_DEVICE(82572EI), 188 189 EM_EMX_DEVICE(82573E), 190 EM_EMX_DEVICE(82573E_IAMT), 191 EM_EMX_DEVICE(82573L), 192 193 EM_DEVICE(82583V), 194 195 EM_EMX_DEVICE(80003ES2LAN_COPPER_SPT), 196 EM_EMX_DEVICE(80003ES2LAN_SERDES_SPT), 197 EM_EMX_DEVICE(80003ES2LAN_COPPER_DPT), 198 EM_EMX_DEVICE(80003ES2LAN_SERDES_DPT), 199 200 EM_DEVICE(ICH8_IGP_M_AMT), 201 EM_DEVICE(ICH8_IGP_AMT), 202 EM_DEVICE(ICH8_IGP_C), 203 EM_DEVICE(ICH8_IFE), 204 EM_DEVICE(ICH8_IFE_GT), 205 EM_DEVICE(ICH8_IFE_G), 206 EM_DEVICE(ICH8_IGP_M), 207 EM_DEVICE(ICH8_82567V_3), 208 209 EM_DEVICE(ICH9_IGP_M_AMT), 210 EM_DEVICE(ICH9_IGP_AMT), 211 EM_DEVICE(ICH9_IGP_C), 212 EM_DEVICE(ICH9_IGP_M), 213 EM_DEVICE(ICH9_IGP_M_V), 214 EM_DEVICE(ICH9_IFE), 215 EM_DEVICE(ICH9_IFE_GT), 216 EM_DEVICE(ICH9_IFE_G), 217 EM_DEVICE(ICH9_BM), 218 219 EM_EMX_DEVICE(82574L), 220 EM_EMX_DEVICE(82574LA), 221 222 EM_DEVICE(ICH10_R_BM_LM), 223 EM_DEVICE(ICH10_R_BM_LF), 224 EM_DEVICE(ICH10_R_BM_V), 225 EM_DEVICE(ICH10_D_BM_LM), 226 EM_DEVICE(ICH10_D_BM_LF), 227 EM_DEVICE(ICH10_D_BM_V), 228 229 EM_DEVICE(PCH_M_HV_LM), 230 EM_DEVICE(PCH_M_HV_LC), 231 EM_DEVICE(PCH_D_HV_DM), 232 EM_DEVICE(PCH_D_HV_DC), 233 234 EM_DEVICE(PCH2_LV_LM), 235 EM_DEVICE(PCH2_LV_V), 236 237 EM_EMX_DEVICE(PCH_LPT_I217_LM), 238 EM_EMX_DEVICE(PCH_LPT_I217_V), 239 EM_EMX_DEVICE(PCH_LPTLP_I218_LM), 240 EM_EMX_DEVICE(PCH_LPTLP_I218_V), 241 EM_EMX_DEVICE(PCH_I218_LM2), 242 EM_EMX_DEVICE(PCH_I218_V2), 243 EM_EMX_DEVICE(PCH_I218_LM3), 244 EM_EMX_DEVICE(PCH_I218_V3), 245 EM_EMX_DEVICE(PCH_SPT_I219_LM), 246 EM_EMX_DEVICE(PCH_SPT_I219_V), 247 EM_EMX_DEVICE(PCH_SPT_I219_LM2), 248 EM_EMX_DEVICE(PCH_SPT_I219_V2), 249 EM_EMX_DEVICE(PCH_SPT_I219_LM3), 250 EM_EMX_DEVICE(PCH_SPT_I219_LM4), 251 EM_EMX_DEVICE(PCH_SPT_I219_V4), 252 EM_EMX_DEVICE(PCH_SPT_I219_LM5), 253 EM_EMX_DEVICE(PCH_SPT_I219_V5), 254 255 /* required last entry */ 256 EM_DEVICE_NULL 257 }; 258 259 static int em_probe(device_t); 260 static int em_attach(device_t); 261 static int em_detach(device_t); 262 static int em_shutdown(device_t); 263 static int em_suspend(device_t); 264 static int em_resume(device_t); 265 266 static void em_init(void *); 267 static void em_stop(struct adapter *); 268 static int em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *); 269 static void em_start(struct ifnet *, struct ifaltq_subque *); 270 #ifdef IFPOLL_ENABLE 271 static void em_npoll(struct ifnet *, struct ifpoll_info *); 272 static void em_npoll_compat(struct ifnet *, void *, int); 273 #endif 274 static void em_watchdog(struct ifnet *); 275 static void em_media_status(struct ifnet *, struct ifmediareq *); 276 static int em_media_change(struct ifnet *); 277 static void em_timer(void *); 278 279 static void em_intr(void *); 280 static void em_intr_mask(void *); 281 static void em_intr_body(struct adapter *, boolean_t); 282 static void em_rxeof(struct adapter *, int); 283 static void em_txeof(struct adapter *); 284 static void em_tx_collect(struct adapter *, boolean_t); 285 static void em_tx_purge(struct adapter *); 286 static void em_txgc_timer(void *); 287 static void em_enable_intr(struct adapter *); 288 static void em_disable_intr(struct adapter *); 289 290 static int em_dma_malloc(struct adapter *, bus_size_t, 291 struct em_dma_alloc *); 292 static void em_dma_free(struct adapter *, struct em_dma_alloc *); 293 static void em_init_tx_ring(struct adapter *); 294 static int em_init_rx_ring(struct adapter *); 295 static int em_create_tx_ring(struct adapter *); 296 static int em_create_rx_ring(struct adapter *); 297 static void em_destroy_tx_ring(struct adapter *, int); 298 static void em_destroy_rx_ring(struct adapter *, int); 299 static int em_newbuf(struct adapter *, int, int); 300 static int em_encap(struct adapter *, struct mbuf **, int *, int *); 301 static void em_rxcsum(struct adapter *, struct e1000_rx_desc *, 302 struct mbuf *); 303 static int em_txcsum(struct adapter *, struct mbuf *, 304 uint32_t *, uint32_t *); 305 static int em_tso_pullup(struct adapter *, struct mbuf **); 306 static int em_tso_setup(struct adapter *, struct mbuf *, 307 uint32_t *, uint32_t *); 308 309 static int em_get_hw_info(struct adapter *); 310 static int em_is_valid_eaddr(const uint8_t *); 311 static int em_alloc_pci_res(struct adapter *); 312 static void em_free_pci_res(struct adapter *); 313 static int em_reset(struct adapter *); 314 static void em_setup_ifp(struct adapter *); 315 static void em_init_tx_unit(struct adapter *); 316 static void em_init_rx_unit(struct adapter *); 317 static void em_update_stats(struct adapter *); 318 static void em_set_promisc(struct adapter *); 319 static void em_disable_promisc(struct adapter *); 320 static void em_set_multi(struct adapter *); 321 static void em_update_link_status(struct adapter *); 322 static void em_smartspeed(struct adapter *); 323 static void em_set_itr(struct adapter *, uint32_t); 324 static void em_disable_aspm(struct adapter *); 325 326 /* Hardware workarounds */ 327 static int em_82547_fifo_workaround(struct adapter *, int); 328 static void em_82547_update_fifo_head(struct adapter *, int); 329 static int em_82547_tx_fifo_reset(struct adapter *); 330 static void em_82547_move_tail(void *); 331 static void em_82547_move_tail_serialized(struct adapter *); 332 static uint32_t em_82544_fill_desc(bus_addr_t, uint32_t, PDESC_ARRAY); 333 334 static void em_print_debug_info(struct adapter *); 335 static void em_print_nvm_info(struct adapter *); 336 static void em_print_hw_stats(struct adapter *); 337 338 static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 339 static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 340 static int em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS); 341 static int em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS); 342 static void em_add_sysctl(struct adapter *adapter); 343 344 /* Management and WOL Support */ 345 static void em_get_mgmt(struct adapter *); 346 static void em_rel_mgmt(struct adapter *); 347 static void em_get_hw_control(struct adapter *); 348 static void em_rel_hw_control(struct adapter *); 349 static void em_enable_wol(device_t); 350 351 static device_method_t em_methods[] = { 352 /* Device interface */ 353 DEVMETHOD(device_probe, em_probe), 354 DEVMETHOD(device_attach, em_attach), 355 DEVMETHOD(device_detach, em_detach), 356 DEVMETHOD(device_shutdown, em_shutdown), 357 DEVMETHOD(device_suspend, em_suspend), 358 DEVMETHOD(device_resume, em_resume), 359 DEVMETHOD_END 360 }; 361 362 static driver_t em_driver = { 363 "em", 364 em_methods, 365 sizeof(struct adapter), 366 }; 367 368 static devclass_t em_devclass; 369 370 DECLARE_DUMMY_MODULE(if_em); 371 MODULE_DEPEND(em, ig_hal, 1, 1, 1); 372 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, NULL, NULL); 373 374 /* 375 * Tunables 376 */ 377 static int em_int_throttle_ceil = EM_DEFAULT_ITR; 378 static int em_rxd = EM_DEFAULT_RXD; 379 static int em_txd = EM_DEFAULT_TXD; 380 static int em_smart_pwr_down = 0; 381 382 /* Controls whether promiscuous also shows bad packets */ 383 static int em_debug_sbp = FALSE; 384 385 static int em_82573_workaround = 1; 386 static int em_msi_enable = 1; 387 388 static char em_flowctrl[IFM_ETH_FC_STRLEN] = IFM_ETH_FC_NONE; 389 390 TUNABLE_INT("hw.em.int_throttle_ceil", &em_int_throttle_ceil); 391 TUNABLE_INT("hw.em.rxd", &em_rxd); 392 TUNABLE_INT("hw.em.txd", &em_txd); 393 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 394 TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 395 TUNABLE_INT("hw.em.82573_workaround", &em_82573_workaround); 396 TUNABLE_INT("hw.em.msi.enable", &em_msi_enable); 397 TUNABLE_STR("hw.em.flow_ctrl", em_flowctrl, sizeof(em_flowctrl)); 398 399 /* Global used in WOL setup with multiport cards */ 400 static int em_global_quad_port_a = 0; 401 402 /* Set this to one to display debug statistics */ 403 static int em_display_debug_stats = 0; 404 405 #if !defined(KTR_IF_EM) 406 #define KTR_IF_EM KTR_ALL 407 #endif 408 KTR_INFO_MASTER(if_em); 409 KTR_INFO(KTR_IF_EM, if_em, intr_beg, 0, "intr begin"); 410 KTR_INFO(KTR_IF_EM, if_em, intr_end, 1, "intr end"); 411 KTR_INFO(KTR_IF_EM, if_em, pkt_receive, 4, "rx packet"); 412 KTR_INFO(KTR_IF_EM, if_em, pkt_txqueue, 5, "tx packet"); 413 KTR_INFO(KTR_IF_EM, if_em, pkt_txclean, 6, "tx clean"); 414 #define logif(name) KTR_LOG(if_em_ ## name) 415 416 static __inline void 417 em_tx_intr(struct adapter *adapter) 418 { 419 struct ifnet *ifp = &adapter->arpcom.ac_if; 420 421 em_txeof(adapter); 422 if (!ifq_is_empty(&ifp->if_snd)) 423 if_devstart(ifp); 424 } 425 426 static __inline void 427 em_free_txbuffer(struct adapter *adapter, struct em_buffer *tx_buffer) 428 { 429 430 KKASSERT(tx_buffer->m_head != NULL); 431 KKASSERT(adapter->tx_nmbuf > 0); 432 adapter->tx_nmbuf--; 433 434 bus_dmamap_unload(adapter->txtag, tx_buffer->map); 435 m_freem(tx_buffer->m_head); 436 tx_buffer->m_head = NULL; 437 } 438 439 static __inline void 440 em_try_txgc(struct adapter *adapter, int dec) 441 { 442 443 if (adapter->tx_running > 0) { 444 adapter->tx_running -= dec; 445 if (adapter->tx_running <= 0 && adapter->tx_nmbuf && 446 adapter->num_tx_desc_avail < adapter->num_tx_desc && 447 adapter->num_tx_desc_avail + adapter->tx_int_nsegs > 448 adapter->num_tx_desc) 449 em_tx_collect(adapter, TRUE); 450 } 451 } 452 453 static void 454 em_txgc_timer(void *xadapter) 455 { 456 struct adapter *adapter = xadapter; 457 struct ifnet *ifp = &adapter->arpcom.ac_if; 458 459 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) != 460 (IFF_RUNNING | IFF_UP)) 461 return; 462 463 if (!lwkt_serialize_try(ifp->if_serializer)) 464 goto done; 465 466 if ((ifp->if_flags & (IFF_RUNNING | IFF_UP | IFF_NPOLLING)) != 467 (IFF_RUNNING | IFF_UP)) { 468 lwkt_serialize_exit(ifp->if_serializer); 469 return; 470 } 471 em_try_txgc(adapter, EM_TX_RUNNING_DEC); 472 473 lwkt_serialize_exit(ifp->if_serializer); 474 done: 475 callout_reset(&adapter->tx_gc_timer, 1, em_txgc_timer, adapter); 476 } 477 478 static int 479 em_probe(device_t dev) 480 { 481 const struct em_vendor_info *ent; 482 uint16_t vid, did; 483 484 vid = pci_get_vendor(dev); 485 did = pci_get_device(dev); 486 487 for (ent = em_vendor_info_array; ent->desc != NULL; ++ent) { 488 if (vid == ent->vendor_id && did == ent->device_id) { 489 device_set_desc(dev, ent->desc); 490 device_set_async_attach(dev, TRUE); 491 return (ent->ret); 492 } 493 } 494 return (ENXIO); 495 } 496 497 static int 498 em_attach(device_t dev) 499 { 500 struct adapter *adapter = device_get_softc(dev); 501 struct ifnet *ifp = &adapter->arpcom.ac_if; 502 int tsize, rsize; 503 int error = 0; 504 uint16_t eeprom_data, device_id, apme_mask; 505 driver_intr_t *intr_func; 506 char flowctrl[IFM_ETH_FC_STRLEN]; 507 508 adapter->dev = adapter->osdep.dev = dev; 509 510 callout_init_mp(&adapter->timer); 511 callout_init_mp(&adapter->tx_fifo_timer); 512 callout_init_mp(&adapter->tx_gc_timer); 513 514 ifmedia_init(&adapter->media, IFM_IMASK | IFM_ETH_FCMASK, 515 em_media_change, em_media_status); 516 517 /* Determine hardware and mac info */ 518 error = em_get_hw_info(adapter); 519 if (error) { 520 device_printf(dev, "Identify hardware failed\n"); 521 goto fail; 522 } 523 524 /* Setup PCI resources */ 525 error = em_alloc_pci_res(adapter); 526 if (error) { 527 device_printf(dev, "Allocation of PCI resources failed\n"); 528 goto fail; 529 } 530 531 /* 532 * For ICH8 and family we need to map the flash memory, 533 * and this must happen after the MAC is identified. 534 * 535 * (SPT does not map the flash with a separate BAR) 536 */ 537 if (adapter->hw.mac.type == e1000_ich8lan || 538 adapter->hw.mac.type == e1000_ich9lan || 539 adapter->hw.mac.type == e1000_ich10lan || 540 adapter->hw.mac.type == e1000_pchlan || 541 adapter->hw.mac.type == e1000_pch2lan || 542 adapter->hw.mac.type == e1000_pch_lpt) { 543 adapter->flash_rid = EM_BAR_FLASH; 544 545 adapter->flash = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 546 &adapter->flash_rid, RF_ACTIVE); 547 if (adapter->flash == NULL) { 548 device_printf(dev, "Mapping of Flash failed\n"); 549 error = ENXIO; 550 goto fail; 551 } 552 adapter->osdep.flash_bus_space_tag = 553 rman_get_bustag(adapter->flash); 554 adapter->osdep.flash_bus_space_handle = 555 rman_get_bushandle(adapter->flash); 556 557 /* 558 * This is used in the shared code 559 * XXX this goof is actually not used. 560 */ 561 adapter->hw.flash_address = (uint8_t *)adapter->flash; 562 } 563 564 switch (adapter->hw.mac.type) { 565 case e1000_82571: 566 case e1000_82572: 567 case e1000_pch_lpt: 568 case e1000_pch_spt: 569 /* 570 * Pullup extra 4bytes into the first data segment for 571 * TSO, see: 572 * 82571/82572 specification update errata #7 573 * 574 * Same applies to I217 (and maybe I218 and I219). 575 * 576 * NOTE: 577 * 4bytes instead of 2bytes, which are mentioned in the 578 * errata, are pulled; mainly to keep rest of the data 579 * properly aligned. 580 */ 581 adapter->flags |= EM_FLAG_TSO_PULLEX; 582 /* FALL THROUGH */ 583 584 default: 585 if (pci_is_pcie(dev)) 586 adapter->flags |= EM_FLAG_TSO; 587 break; 588 } 589 590 /* Do Shared Code initialization */ 591 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 592 device_printf(dev, "Setup of Shared code failed\n"); 593 error = ENXIO; 594 goto fail; 595 } 596 597 e1000_get_bus_info(&adapter->hw); 598 599 /* 600 * Validate number of transmit and receive descriptors. It 601 * must not exceed hardware maximum, and must be multiple 602 * of E1000_DBA_ALIGN. 603 */ 604 if ((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN != 0 || 605 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) || 606 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) || 607 em_txd < EM_MIN_TXD) { 608 if (adapter->hw.mac.type < e1000_82544) 609 adapter->num_tx_desc = EM_MAX_TXD_82543; 610 else 611 adapter->num_tx_desc = EM_DEFAULT_TXD; 612 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 613 adapter->num_tx_desc, em_txd); 614 } else { 615 adapter->num_tx_desc = em_txd; 616 } 617 if ((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN != 0 || 618 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) || 619 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) || 620 em_rxd < EM_MIN_RXD) { 621 if (adapter->hw.mac.type < e1000_82544) 622 adapter->num_rx_desc = EM_MAX_RXD_82543; 623 else 624 adapter->num_rx_desc = EM_DEFAULT_RXD; 625 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 626 adapter->num_rx_desc, em_rxd); 627 } else { 628 adapter->num_rx_desc = em_rxd; 629 } 630 631 adapter->hw.mac.autoneg = DO_AUTO_NEG; 632 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 633 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 634 adapter->rx_buffer_len = MCLBYTES; 635 636 /* 637 * Interrupt throttle rate 638 */ 639 if (em_int_throttle_ceil == 0) { 640 adapter->int_throttle_ceil = 0; 641 } else { 642 int throttle = em_int_throttle_ceil; 643 644 if (throttle < 0) 645 throttle = EM_DEFAULT_ITR; 646 647 /* Recalculate the tunable value to get the exact frequency. */ 648 throttle = 1000000000 / 256 / throttle; 649 650 /* Upper 16bits of ITR is reserved and should be zero */ 651 if (throttle & 0xffff0000) 652 throttle = 1000000000 / 256 / EM_DEFAULT_ITR; 653 654 adapter->int_throttle_ceil = 1000000000 / 256 / throttle; 655 } 656 657 e1000_init_script_state_82541(&adapter->hw, TRUE); 658 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); 659 660 /* Copper options */ 661 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 662 adapter->hw.phy.mdix = AUTO_ALL_MODES; 663 adapter->hw.phy.disable_polarity_correction = FALSE; 664 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 665 } 666 667 /* Set the frame limits assuming standard ethernet sized frames. */ 668 adapter->hw.mac.max_frame_size = 669 ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN; 670 adapter->min_frame_size = ETH_ZLEN + ETHER_CRC_LEN; 671 672 /* This controls when hardware reports transmit completion status. */ 673 adapter->hw.mac.report_tx_early = 1; 674 675 /* 676 * Create top level busdma tag 677 */ 678 error = bus_dma_tag_create(NULL, 1, 0, 679 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 680 NULL, NULL, 681 BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT, 682 0, &adapter->parent_dtag); 683 if (error) { 684 device_printf(dev, "could not create top level DMA tag\n"); 685 goto fail; 686 } 687 688 /* 689 * Allocate Transmit Descriptor ring 690 */ 691 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), 692 EM_DBA_ALIGN); 693 error = em_dma_malloc(adapter, tsize, &adapter->txdma); 694 if (error) { 695 device_printf(dev, "Unable to allocate tx_desc memory\n"); 696 goto fail; 697 } 698 adapter->tx_desc_base = adapter->txdma.dma_vaddr; 699 700 /* 701 * Allocate Receive Descriptor ring 702 */ 703 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc), 704 EM_DBA_ALIGN); 705 error = em_dma_malloc(adapter, rsize, &adapter->rxdma); 706 if (error) { 707 device_printf(dev, "Unable to allocate rx_desc memory\n"); 708 goto fail; 709 } 710 adapter->rx_desc_base = adapter->rxdma.dma_vaddr; 711 712 /* Allocate multicast array memory. */ 713 adapter->mta = kmalloc(ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES, 714 M_DEVBUF, M_WAITOK); 715 716 /* Indicate SOL/IDER usage */ 717 if (e1000_check_reset_block(&adapter->hw)) { 718 device_printf(dev, 719 "PHY reset is blocked due to SOL/IDER session.\n"); 720 } 721 722 /* Disable EEE */ 723 adapter->hw.dev_spec.ich8lan.eee_disable = 1; 724 725 /* 726 * Start from a known state, this is important in reading the 727 * nvm and mac from that. 728 */ 729 e1000_reset_hw(&adapter->hw); 730 731 /* Make sure we have a good EEPROM before we read from it */ 732 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 733 /* 734 * Some PCI-E parts fail the first check due to 735 * the link being in sleep state, call it again, 736 * if it fails a second time its a real issue. 737 */ 738 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 739 device_printf(dev, 740 "The EEPROM Checksum Is Not Valid\n"); 741 error = EIO; 742 goto fail; 743 } 744 } 745 746 /* Copy the permanent MAC address out of the EEPROM */ 747 if (e1000_read_mac_addr(&adapter->hw) < 0) { 748 device_printf(dev, "EEPROM read error while reading MAC" 749 " address\n"); 750 error = EIO; 751 goto fail; 752 } 753 if (!em_is_valid_eaddr(adapter->hw.mac.addr)) { 754 device_printf(dev, "Invalid MAC address\n"); 755 error = EIO; 756 goto fail; 757 } 758 759 /* Disable ULP support */ 760 e1000_disable_ulp_lpt_lp(&adapter->hw, TRUE); 761 762 /* Allocate transmit descriptors and buffers */ 763 error = em_create_tx_ring(adapter); 764 if (error) { 765 device_printf(dev, "Could not setup transmit structures\n"); 766 goto fail; 767 } 768 769 /* Allocate receive descriptors and buffers */ 770 error = em_create_rx_ring(adapter); 771 if (error) { 772 device_printf(dev, "Could not setup receive structures\n"); 773 goto fail; 774 } 775 776 /* Manually turn off all interrupts */ 777 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 778 779 /* Determine if we have to control management hardware */ 780 if (e1000_enable_mng_pass_thru(&adapter->hw)) 781 adapter->flags |= EM_FLAG_HAS_MGMT; 782 783 /* 784 * Setup Wake-on-Lan 785 */ 786 apme_mask = EM_EEPROM_APME; 787 eeprom_data = 0; 788 switch (adapter->hw.mac.type) { 789 case e1000_82542: 790 case e1000_82543: 791 break; 792 793 case e1000_82573: 794 case e1000_82583: 795 adapter->flags |= EM_FLAG_HAS_AMT; 796 /* FALL THROUGH */ 797 798 case e1000_82546: 799 case e1000_82546_rev_3: 800 case e1000_82571: 801 case e1000_82572: 802 case e1000_80003es2lan: 803 if (adapter->hw.bus.func == 1) { 804 e1000_read_nvm(&adapter->hw, 805 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 806 } else { 807 e1000_read_nvm(&adapter->hw, 808 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 809 } 810 break; 811 812 case e1000_ich8lan: 813 case e1000_ich9lan: 814 case e1000_ich10lan: 815 case e1000_pchlan: 816 case e1000_pch2lan: 817 apme_mask = E1000_WUC_APME; 818 adapter->flags |= EM_FLAG_HAS_AMT; 819 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); 820 break; 821 822 default: 823 e1000_read_nvm(&adapter->hw, 824 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 825 break; 826 } 827 if (eeprom_data & apme_mask) 828 adapter->wol = E1000_WUFC_MAG | E1000_WUFC_MC; 829 830 /* 831 * We have the eeprom settings, now apply the special cases 832 * where the eeprom may be wrong or the board won't support 833 * wake on lan on a particular port 834 */ 835 device_id = pci_get_device(dev); 836 switch (device_id) { 837 case E1000_DEV_ID_82546GB_PCIE: 838 adapter->wol = 0; 839 break; 840 841 case E1000_DEV_ID_82546EB_FIBER: 842 case E1000_DEV_ID_82546GB_FIBER: 843 case E1000_DEV_ID_82571EB_FIBER: 844 /* 845 * Wake events only supported on port A for dual fiber 846 * regardless of eeprom setting 847 */ 848 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 849 E1000_STATUS_FUNC_1) 850 adapter->wol = 0; 851 break; 852 853 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 854 case E1000_DEV_ID_82571EB_QUAD_COPPER: 855 case E1000_DEV_ID_82571EB_QUAD_FIBER: 856 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 857 /* if quad port adapter, disable WoL on all but port A */ 858 if (em_global_quad_port_a != 0) 859 adapter->wol = 0; 860 /* Reset for multiple quad port adapters */ 861 if (++em_global_quad_port_a == 4) 862 em_global_quad_port_a = 0; 863 break; 864 } 865 866 /* XXX disable wol */ 867 adapter->wol = 0; 868 869 /* Setup flow control. */ 870 device_getenv_string(dev, "flow_ctrl", flowctrl, sizeof(flowctrl), 871 em_flowctrl); 872 adapter->ifm_flowctrl = ifmedia_str2ethfc(flowctrl); 873 if (adapter->hw.mac.type == e1000_pchlan) { 874 /* Only PAUSE reception is supported on PCH */ 875 adapter->ifm_flowctrl &= ~IFM_ETH_TXPAUSE; 876 } 877 878 /* Setup OS specific network interface */ 879 em_setup_ifp(adapter); 880 881 /* Add sysctl tree, must after em_setup_ifp() */ 882 em_add_sysctl(adapter); 883 884 #ifdef IFPOLL_ENABLE 885 /* Polling setup */ 886 ifpoll_compat_setup(&adapter->npoll, 887 device_get_sysctl_ctx(dev), device_get_sysctl_tree(dev), 888 device_get_unit(dev), ifp->if_serializer); 889 #endif 890 891 /* Reset the hardware */ 892 error = em_reset(adapter); 893 if (error) { 894 /* 895 * Some 82573 parts fail the first reset, call it again, 896 * if it fails a second time its a real issue. 897 */ 898 error = em_reset(adapter); 899 if (error) { 900 device_printf(dev, "Unable to reset the hardware\n"); 901 ether_ifdetach(ifp); 902 goto fail; 903 } 904 } 905 906 /* Initialize statistics */ 907 em_update_stats(adapter); 908 909 adapter->hw.mac.get_link_status = 1; 910 em_update_link_status(adapter); 911 912 /* Do we need workaround for 82544 PCI-X adapter? */ 913 if (adapter->hw.bus.type == e1000_bus_type_pcix && 914 adapter->hw.mac.type == e1000_82544) 915 adapter->pcix_82544 = TRUE; 916 else 917 adapter->pcix_82544 = FALSE; 918 919 if (adapter->pcix_82544) { 920 /* 921 * 82544 on PCI-X may split one TX segment 922 * into two TX descs, so we double its number 923 * of spare TX desc here. 924 */ 925 adapter->spare_tx_desc = 2 * EM_TX_SPARE; 926 } else { 927 adapter->spare_tx_desc = EM_TX_SPARE; 928 } 929 if (adapter->flags & EM_FLAG_TSO) 930 adapter->spare_tx_desc = EM_TX_SPARE_TSO; 931 adapter->tx_wreg_nsegs = EM_DEFAULT_TXWREG; 932 933 /* 934 * Keep following relationship between spare_tx_desc, oact_tx_desc 935 * and tx_int_nsegs: 936 * (spare_tx_desc + EM_TX_RESERVED) <= 937 * oact_tx_desc <= EM_TX_OACTIVE_MAX <= tx_int_nsegs 938 */ 939 adapter->oact_tx_desc = adapter->num_tx_desc / 8; 940 if (adapter->oact_tx_desc > EM_TX_OACTIVE_MAX) 941 adapter->oact_tx_desc = EM_TX_OACTIVE_MAX; 942 if (adapter->oact_tx_desc < adapter->spare_tx_desc + EM_TX_RESERVED) 943 adapter->oact_tx_desc = adapter->spare_tx_desc + EM_TX_RESERVED; 944 945 adapter->tx_int_nsegs = adapter->num_tx_desc / 16; 946 if (adapter->tx_int_nsegs < adapter->oact_tx_desc) 947 adapter->tx_int_nsegs = adapter->oact_tx_desc; 948 949 /* Non-AMT based hardware can now take control from firmware */ 950 if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) == 951 EM_FLAG_HAS_MGMT && adapter->hw.mac.type >= e1000_82571) 952 em_get_hw_control(adapter); 953 954 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res)); 955 956 /* 957 * Missing Interrupt Following ICR read: 958 * 959 * 82571/82572 specification update errata #76 960 * 82573 specification update errata #31 961 * 82574 specification update errata #12 962 * 82583 specification update errata #4 963 */ 964 intr_func = em_intr; 965 if ((adapter->flags & EM_FLAG_SHARED_INTR) && 966 (adapter->hw.mac.type == e1000_82571 || 967 adapter->hw.mac.type == e1000_82572 || 968 adapter->hw.mac.type == e1000_82573 || 969 adapter->hw.mac.type == e1000_82574 || 970 adapter->hw.mac.type == e1000_82583)) 971 intr_func = em_intr_mask; 972 973 error = bus_setup_intr(dev, adapter->intr_res, INTR_MPSAFE, 974 intr_func, adapter, &adapter->intr_tag, 975 ifp->if_serializer); 976 if (error) { 977 device_printf(dev, "Failed to register interrupt handler"); 978 ether_ifdetach(ifp); 979 goto fail; 980 } 981 return (0); 982 fail: 983 em_detach(dev); 984 return (error); 985 } 986 987 static int 988 em_detach(device_t dev) 989 { 990 struct adapter *adapter = device_get_softc(dev); 991 992 if (device_is_attached(dev)) { 993 struct ifnet *ifp = &adapter->arpcom.ac_if; 994 995 lwkt_serialize_enter(ifp->if_serializer); 996 997 em_stop(adapter); 998 999 e1000_phy_hw_reset(&adapter->hw); 1000 1001 em_rel_mgmt(adapter); 1002 em_rel_hw_control(adapter); 1003 1004 if (adapter->wol) { 1005 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 1006 E1000_WUC_PME_EN); 1007 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 1008 em_enable_wol(dev); 1009 } 1010 1011 bus_teardown_intr(dev, adapter->intr_res, adapter->intr_tag); 1012 1013 lwkt_serialize_exit(ifp->if_serializer); 1014 1015 ether_ifdetach(ifp); 1016 } else if (adapter->memory != NULL) { 1017 em_rel_hw_control(adapter); 1018 } 1019 1020 ifmedia_removeall(&adapter->media); 1021 bus_generic_detach(dev); 1022 1023 em_free_pci_res(adapter); 1024 1025 em_destroy_tx_ring(adapter, adapter->num_tx_desc); 1026 em_destroy_rx_ring(adapter, adapter->num_rx_desc); 1027 1028 /* Free Transmit Descriptor ring */ 1029 if (adapter->tx_desc_base) 1030 em_dma_free(adapter, &adapter->txdma); 1031 1032 /* Free Receive Descriptor ring */ 1033 if (adapter->rx_desc_base) 1034 em_dma_free(adapter, &adapter->rxdma); 1035 1036 /* Free top level busdma tag */ 1037 if (adapter->parent_dtag != NULL) 1038 bus_dma_tag_destroy(adapter->parent_dtag); 1039 1040 if (adapter->mta != NULL) 1041 kfree(adapter->mta, M_DEVBUF); 1042 1043 return (0); 1044 } 1045 1046 static int 1047 em_shutdown(device_t dev) 1048 { 1049 return em_suspend(dev); 1050 } 1051 1052 static int 1053 em_suspend(device_t dev) 1054 { 1055 struct adapter *adapter = device_get_softc(dev); 1056 struct ifnet *ifp = &adapter->arpcom.ac_if; 1057 1058 lwkt_serialize_enter(ifp->if_serializer); 1059 1060 em_stop(adapter); 1061 1062 em_rel_mgmt(adapter); 1063 em_rel_hw_control(adapter); 1064 1065 if (adapter->wol) { 1066 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 1067 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 1068 em_enable_wol(dev); 1069 } 1070 1071 lwkt_serialize_exit(ifp->if_serializer); 1072 1073 return bus_generic_suspend(dev); 1074 } 1075 1076 static int 1077 em_resume(device_t dev) 1078 { 1079 struct adapter *adapter = device_get_softc(dev); 1080 struct ifnet *ifp = &adapter->arpcom.ac_if; 1081 1082 lwkt_serialize_enter(ifp->if_serializer); 1083 1084 if (adapter->hw.mac.type == e1000_pch2lan) 1085 e1000_resume_workarounds_pchlan(&adapter->hw); 1086 1087 em_init(adapter); 1088 em_get_mgmt(adapter); 1089 if_devstart(ifp); 1090 1091 lwkt_serialize_exit(ifp->if_serializer); 1092 1093 return bus_generic_resume(dev); 1094 } 1095 1096 static void 1097 em_start(struct ifnet *ifp, struct ifaltq_subque *ifsq) 1098 { 1099 struct adapter *adapter = ifp->if_softc; 1100 struct mbuf *m_head; 1101 int idx = -1, nsegs = 0; 1102 1103 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq); 1104 ASSERT_SERIALIZED(ifp->if_serializer); 1105 1106 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd)) 1107 return; 1108 1109 if (!adapter->link_active) { 1110 ifq_purge(&ifp->if_snd); 1111 return; 1112 } 1113 1114 while (!ifq_is_empty(&ifp->if_snd)) { 1115 /* Now do we at least have a minimal? */ 1116 if (EM_IS_OACTIVE(adapter)) { 1117 em_tx_collect(adapter, FALSE); 1118 if (EM_IS_OACTIVE(adapter)) { 1119 ifq_set_oactive(&ifp->if_snd); 1120 adapter->no_tx_desc_avail1++; 1121 break; 1122 } 1123 } 1124 1125 logif(pkt_txqueue); 1126 m_head = ifq_dequeue(&ifp->if_snd); 1127 if (m_head == NULL) 1128 break; 1129 1130 if (em_encap(adapter, &m_head, &nsegs, &idx)) { 1131 IFNET_STAT_INC(ifp, oerrors, 1); 1132 em_tx_collect(adapter, FALSE); 1133 continue; 1134 } 1135 1136 /* 1137 * TX interrupt are aggressively aggregated, so increasing 1138 * opackets at TX interrupt time will make the opackets 1139 * statistics vastly inaccurate; we do the opackets increment 1140 * now. 1141 */ 1142 IFNET_STAT_INC(ifp, opackets, 1); 1143 1144 if (nsegs >= adapter->tx_wreg_nsegs && idx >= 0) { 1145 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx); 1146 nsegs = 0; 1147 idx = -1; 1148 } 1149 1150 /* Send a copy of the frame to the BPF listener */ 1151 ETHER_BPF_MTAP(ifp, m_head); 1152 1153 /* Set timeout in case hardware has problems transmitting. */ 1154 ifp->if_timer = EM_TX_TIMEOUT; 1155 } 1156 if (idx >= 0) 1157 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx); 1158 adapter->tx_running = EM_TX_RUNNING; 1159 } 1160 1161 static int 1162 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr) 1163 { 1164 struct adapter *adapter = ifp->if_softc; 1165 struct ifreq *ifr = (struct ifreq *)data; 1166 uint16_t eeprom_data = 0; 1167 int max_frame_size, mask, reinit; 1168 int error = 0; 1169 1170 ASSERT_SERIALIZED(ifp->if_serializer); 1171 1172 switch (command) { 1173 case SIOCSIFMTU: 1174 switch (adapter->hw.mac.type) { 1175 case e1000_82573: 1176 /* 1177 * 82573 only supports jumbo frames 1178 * if ASPM is disabled. 1179 */ 1180 e1000_read_nvm(&adapter->hw, 1181 NVM_INIT_3GIO_3, 1, &eeprom_data); 1182 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1183 max_frame_size = ETHER_MAX_LEN; 1184 break; 1185 } 1186 /* FALL THROUGH */ 1187 1188 /* Limit Jumbo Frame size */ 1189 case e1000_82571: 1190 case e1000_82572: 1191 case e1000_ich9lan: 1192 case e1000_ich10lan: 1193 case e1000_pch2lan: 1194 case e1000_pch_lpt: 1195 case e1000_pch_spt: 1196 case e1000_82574: 1197 case e1000_82583: 1198 case e1000_80003es2lan: 1199 max_frame_size = 9234; 1200 break; 1201 1202 case e1000_pchlan: 1203 max_frame_size = 4096; 1204 break; 1205 1206 /* Adapters that do not support jumbo frames */ 1207 case e1000_82542: 1208 case e1000_ich8lan: 1209 max_frame_size = ETHER_MAX_LEN; 1210 break; 1211 1212 default: 1213 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1214 break; 1215 } 1216 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1217 ETHER_CRC_LEN) { 1218 error = EINVAL; 1219 break; 1220 } 1221 1222 ifp->if_mtu = ifr->ifr_mtu; 1223 adapter->hw.mac.max_frame_size = 1224 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1225 1226 if (ifp->if_flags & IFF_RUNNING) 1227 em_init(adapter); 1228 break; 1229 1230 case SIOCSIFFLAGS: 1231 if (ifp->if_flags & IFF_UP) { 1232 if ((ifp->if_flags & IFF_RUNNING)) { 1233 if ((ifp->if_flags ^ adapter->if_flags) & 1234 (IFF_PROMISC | IFF_ALLMULTI)) { 1235 em_disable_promisc(adapter); 1236 em_set_promisc(adapter); 1237 } 1238 } else { 1239 em_init(adapter); 1240 } 1241 } else if (ifp->if_flags & IFF_RUNNING) { 1242 em_stop(adapter); 1243 } 1244 adapter->if_flags = ifp->if_flags; 1245 break; 1246 1247 case SIOCADDMULTI: 1248 case SIOCDELMULTI: 1249 if (ifp->if_flags & IFF_RUNNING) { 1250 em_disable_intr(adapter); 1251 em_set_multi(adapter); 1252 if (adapter->hw.mac.type == e1000_82542 && 1253 adapter->hw.revision_id == E1000_REVISION_2) 1254 em_init_rx_unit(adapter); 1255 #ifdef IFPOLL_ENABLE 1256 if (!(ifp->if_flags & IFF_NPOLLING)) 1257 #endif 1258 em_enable_intr(adapter); 1259 } 1260 break; 1261 1262 case SIOCSIFMEDIA: 1263 /* Check SOL/IDER usage */ 1264 if (e1000_check_reset_block(&adapter->hw)) { 1265 device_printf(adapter->dev, "Media change is" 1266 " blocked due to SOL/IDER session.\n"); 1267 break; 1268 } 1269 /* FALL THROUGH */ 1270 1271 case SIOCGIFMEDIA: 1272 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1273 break; 1274 1275 case SIOCSIFCAP: 1276 reinit = 0; 1277 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1278 if (mask & IFCAP_RXCSUM) { 1279 ifp->if_capenable ^= IFCAP_RXCSUM; 1280 reinit = 1; 1281 } 1282 if (mask & IFCAP_TXCSUM) { 1283 ifp->if_capenable ^= IFCAP_TXCSUM; 1284 if (ifp->if_capenable & IFCAP_TXCSUM) 1285 ifp->if_hwassist |= EM_CSUM_FEATURES; 1286 else 1287 ifp->if_hwassist &= ~EM_CSUM_FEATURES; 1288 } 1289 if (mask & IFCAP_TSO) { 1290 ifp->if_capenable ^= IFCAP_TSO; 1291 if (ifp->if_capenable & IFCAP_TSO) 1292 ifp->if_hwassist |= CSUM_TSO; 1293 else 1294 ifp->if_hwassist &= ~CSUM_TSO; 1295 } 1296 if (mask & IFCAP_VLAN_HWTAGGING) { 1297 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1298 reinit = 1; 1299 } 1300 if (reinit && (ifp->if_flags & IFF_RUNNING)) 1301 em_init(adapter); 1302 break; 1303 1304 default: 1305 error = ether_ioctl(ifp, command, data); 1306 break; 1307 } 1308 return (error); 1309 } 1310 1311 static void 1312 em_watchdog(struct ifnet *ifp) 1313 { 1314 struct adapter *adapter = ifp->if_softc; 1315 1316 ASSERT_SERIALIZED(ifp->if_serializer); 1317 1318 /* 1319 * The timer is set to 5 every time start queues a packet. 1320 * Then txeof keeps resetting it as long as it cleans at 1321 * least one descriptor. 1322 * Finally, anytime all descriptors are clean the timer is 1323 * set to 0. 1324 */ 1325 1326 if (E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == 1327 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) { 1328 /* 1329 * If we reach here, all TX jobs are completed and 1330 * the TX engine should have been idled for some time. 1331 * We don't need to call if_devstart() here. 1332 */ 1333 ifq_clr_oactive(&ifp->if_snd); 1334 ifp->if_timer = 0; 1335 return; 1336 } 1337 1338 /* 1339 * If we are in this routine because of pause frames, then 1340 * don't reset the hardware. 1341 */ 1342 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 1343 E1000_STATUS_TXOFF) { 1344 ifp->if_timer = EM_TX_TIMEOUT; 1345 return; 1346 } 1347 1348 if (e1000_check_for_link(&adapter->hw) == 0) 1349 if_printf(ifp, "watchdog timeout -- resetting\n"); 1350 1351 IFNET_STAT_INC(ifp, oerrors, 1); 1352 adapter->watchdog_events++; 1353 1354 em_init(adapter); 1355 1356 if (!ifq_is_empty(&ifp->if_snd)) 1357 if_devstart(ifp); 1358 } 1359 1360 static void 1361 em_init(void *xsc) 1362 { 1363 struct adapter *adapter = xsc; 1364 struct ifnet *ifp = &adapter->arpcom.ac_if; 1365 device_t dev = adapter->dev; 1366 1367 ASSERT_SERIALIZED(ifp->if_serializer); 1368 1369 em_stop(adapter); 1370 1371 /* Get the latest mac address, User can use a LAA */ 1372 bcopy(IF_LLADDR(ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN); 1373 1374 /* Put the address into the Receive Address Array */ 1375 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1376 1377 /* 1378 * With the 82571 adapter, RAR[0] may be overwritten 1379 * when the other port is reset, we make a duplicate 1380 * in RAR[14] for that eventuality, this assures 1381 * the interface continues to function. 1382 */ 1383 if (adapter->hw.mac.type == e1000_82571) { 1384 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1385 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1386 E1000_RAR_ENTRIES - 1); 1387 } 1388 1389 /* Reset the hardware */ 1390 if (em_reset(adapter)) { 1391 device_printf(dev, "Unable to reset the hardware\n"); 1392 /* XXX em_stop()? */ 1393 return; 1394 } 1395 em_update_link_status(adapter); 1396 1397 /* Setup VLAN support, basic and offload if available */ 1398 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1399 1400 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1401 uint32_t ctrl; 1402 1403 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1404 ctrl |= E1000_CTRL_VME; 1405 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1406 } 1407 1408 /* Configure for OS presence */ 1409 em_get_mgmt(adapter); 1410 1411 /* Prepare transmit descriptors and buffers */ 1412 em_init_tx_ring(adapter); 1413 em_init_tx_unit(adapter); 1414 1415 /* Setup Multicast table */ 1416 em_set_multi(adapter); 1417 1418 /* Prepare receive descriptors and buffers */ 1419 if (em_init_rx_ring(adapter)) { 1420 device_printf(dev, "Could not setup receive structures\n"); 1421 em_stop(adapter); 1422 return; 1423 } 1424 em_init_rx_unit(adapter); 1425 1426 /* Don't lose promiscuous settings */ 1427 em_set_promisc(adapter); 1428 1429 /* Reset hardware counters */ 1430 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1431 1432 /* MSI/X configuration for 82574 */ 1433 if (adapter->hw.mac.type == e1000_82574) { 1434 int tmp; 1435 1436 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1437 tmp |= E1000_CTRL_EXT_PBA_CLR; 1438 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1439 /* 1440 * XXX MSIX 1441 * Set the IVAR - interrupt vector routing. 1442 * Each nibble represents a vector, high bit 1443 * is enable, other 3 bits are the MSIX table 1444 * entry, we map RXQ0 to 0, TXQ0 to 1, and 1445 * Link (other) to 2, hence the magic number. 1446 */ 1447 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908); 1448 } 1449 1450 #ifdef IFPOLL_ENABLE 1451 /* 1452 * Only enable interrupts if we are not polling, make sure 1453 * they are off otherwise. 1454 */ 1455 if (ifp->if_flags & IFF_NPOLLING) 1456 em_disable_intr(adapter); 1457 else 1458 #endif /* IFPOLL_ENABLE */ 1459 em_enable_intr(adapter); 1460 1461 /* AMT based hardware can now take control from firmware */ 1462 if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) == 1463 (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT) && 1464 adapter->hw.mac.type >= e1000_82571) 1465 em_get_hw_control(adapter); 1466 1467 ifp->if_flags |= IFF_RUNNING; 1468 ifq_clr_oactive(&ifp->if_snd); 1469 1470 #ifdef IFPOLL_ENABLE 1471 if ((ifp->if_flags & IFF_NPOLLING) == 0) 1472 #endif 1473 { 1474 callout_reset_bycpu(&adapter->tx_gc_timer, 1, 1475 em_txgc_timer, adapter, 1476 rman_get_cpuid(adapter->intr_res)); 1477 } 1478 callout_reset(&adapter->timer, hz, em_timer, adapter); 1479 } 1480 1481 #ifdef IFPOLL_ENABLE 1482 1483 static void 1484 em_npoll_compat(struct ifnet *ifp, void *arg __unused, int count) 1485 { 1486 struct adapter *adapter = ifp->if_softc; 1487 1488 ASSERT_SERIALIZED(ifp->if_serializer); 1489 1490 if (adapter->npoll.ifpc_stcount-- == 0) { 1491 uint32_t reg_icr; 1492 1493 adapter->npoll.ifpc_stcount = adapter->npoll.ifpc_stfrac; 1494 1495 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1496 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1497 callout_stop(&adapter->timer); 1498 adapter->hw.mac.get_link_status = 1; 1499 em_update_link_status(adapter); 1500 callout_reset(&adapter->timer, hz, em_timer, adapter); 1501 } 1502 } 1503 1504 em_rxeof(adapter, count); 1505 1506 em_tx_intr(adapter); 1507 em_try_txgc(adapter, 1); 1508 } 1509 1510 static void 1511 em_npoll(struct ifnet *ifp, struct ifpoll_info *info) 1512 { 1513 struct adapter *adapter = ifp->if_softc; 1514 1515 ASSERT_SERIALIZED(ifp->if_serializer); 1516 1517 if (info != NULL) { 1518 int cpuid = adapter->npoll.ifpc_cpuid; 1519 1520 info->ifpi_rx[cpuid].poll_func = em_npoll_compat; 1521 info->ifpi_rx[cpuid].arg = NULL; 1522 info->ifpi_rx[cpuid].serializer = ifp->if_serializer; 1523 1524 ifq_set_cpuid(&ifp->if_snd, cpuid); 1525 } else { 1526 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res)); 1527 } 1528 if (ifp->if_flags & IFF_RUNNING) 1529 em_init(adapter); 1530 } 1531 1532 #endif /* IFPOLL_ENABLE */ 1533 1534 static void 1535 em_intr(void *xsc) 1536 { 1537 em_intr_body(xsc, TRUE); 1538 } 1539 1540 static void 1541 em_intr_body(struct adapter *adapter, boolean_t chk_asserted) 1542 { 1543 struct ifnet *ifp = &adapter->arpcom.ac_if; 1544 uint32_t reg_icr; 1545 1546 logif(intr_beg); 1547 ASSERT_SERIALIZED(ifp->if_serializer); 1548 1549 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1550 1551 if (chk_asserted && 1552 ((adapter->hw.mac.type >= e1000_82571 && 1553 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) || 1554 reg_icr == 0)) { 1555 logif(intr_end); 1556 return; 1557 } 1558 1559 /* 1560 * XXX: some laptops trigger several spurious interrupts 1561 * on em(4) when in the resume cycle. The ICR register 1562 * reports all-ones value in this case. Processing such 1563 * interrupts would lead to a freeze. I don't know why. 1564 */ 1565 if (reg_icr == 0xffffffff) { 1566 logif(intr_end); 1567 return; 1568 } 1569 1570 if (ifp->if_flags & IFF_RUNNING) { 1571 if (reg_icr & 1572 (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO)) 1573 em_rxeof(adapter, -1); 1574 if (reg_icr & E1000_ICR_TXDW) 1575 em_tx_intr(adapter); 1576 } 1577 1578 /* Link status change */ 1579 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1580 callout_stop(&adapter->timer); 1581 adapter->hw.mac.get_link_status = 1; 1582 em_update_link_status(adapter); 1583 1584 /* Deal with TX cruft when link lost */ 1585 em_tx_purge(adapter); 1586 1587 callout_reset(&adapter->timer, hz, em_timer, adapter); 1588 } 1589 1590 if (reg_icr & E1000_ICR_RXO) 1591 adapter->rx_overruns++; 1592 1593 logif(intr_end); 1594 } 1595 1596 static void 1597 em_intr_mask(void *xsc) 1598 { 1599 struct adapter *adapter = xsc; 1600 1601 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 1602 /* 1603 * NOTE: 1604 * ICR.INT_ASSERTED bit will never be set if IMS is 0, 1605 * so don't check it. 1606 */ 1607 em_intr_body(adapter, FALSE); 1608 E1000_WRITE_REG(&adapter->hw, E1000_IMS, IMS_ENABLE_MASK); 1609 } 1610 1611 static void 1612 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1613 { 1614 struct adapter *adapter = ifp->if_softc; 1615 1616 ASSERT_SERIALIZED(ifp->if_serializer); 1617 1618 em_update_link_status(adapter); 1619 1620 ifmr->ifm_status = IFM_AVALID; 1621 ifmr->ifm_active = IFM_ETHER; 1622 1623 if (!adapter->link_active) { 1624 if (adapter->hw.mac.autoneg) 1625 ifmr->ifm_active |= IFM_NONE; 1626 else 1627 ifmr->ifm_active = adapter->media.ifm_media; 1628 return; 1629 } 1630 1631 ifmr->ifm_status |= IFM_ACTIVE; 1632 if (adapter->ifm_flowctrl & IFM_ETH_FORCEPAUSE) 1633 ifmr->ifm_active |= adapter->ifm_flowctrl; 1634 1635 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 1636 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 1637 u_char fiber_type = IFM_1000_SX; 1638 1639 if (adapter->hw.mac.type == e1000_82545) 1640 fiber_type = IFM_1000_LX; 1641 ifmr->ifm_active |= fiber_type | IFM_FDX; 1642 } else { 1643 switch (adapter->link_speed) { 1644 case 10: 1645 ifmr->ifm_active |= IFM_10_T; 1646 break; 1647 case 100: 1648 ifmr->ifm_active |= IFM_100_TX; 1649 break; 1650 1651 case 1000: 1652 ifmr->ifm_active |= IFM_1000_T; 1653 break; 1654 } 1655 if (adapter->link_duplex == FULL_DUPLEX) 1656 ifmr->ifm_active |= IFM_FDX; 1657 else 1658 ifmr->ifm_active |= IFM_HDX; 1659 } 1660 if (ifmr->ifm_active & IFM_FDX) { 1661 ifmr->ifm_active |= 1662 e1000_fc2ifmedia(adapter->hw.fc.current_mode); 1663 } 1664 } 1665 1666 static int 1667 em_media_change(struct ifnet *ifp) 1668 { 1669 struct adapter *adapter = ifp->if_softc; 1670 struct ifmedia *ifm = &adapter->media; 1671 1672 ASSERT_SERIALIZED(ifp->if_serializer); 1673 1674 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1675 return (EINVAL); 1676 1677 if (adapter->hw.mac.type == e1000_pchlan && 1678 (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)) { 1679 if (bootverbose) 1680 if_printf(ifp, "TX PAUSE is not supported on PCH\n"); 1681 return EINVAL; 1682 } 1683 1684 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1685 case IFM_AUTO: 1686 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1687 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1688 break; 1689 1690 case IFM_1000_LX: 1691 case IFM_1000_SX: 1692 case IFM_1000_T: 1693 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1694 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1695 break; 1696 1697 case IFM_100_TX: 1698 if (IFM_OPTIONS(ifm->ifm_media) & IFM_FDX) { 1699 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1700 } else { 1701 if (IFM_OPTIONS(ifm->ifm_media) & 1702 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) { 1703 if (bootverbose) { 1704 if_printf(ifp, "Flow control is not " 1705 "allowed for half-duplex\n"); 1706 } 1707 return EINVAL; 1708 } 1709 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1710 } 1711 adapter->hw.mac.autoneg = FALSE; 1712 adapter->hw.phy.autoneg_advertised = 0; 1713 break; 1714 1715 case IFM_10_T: 1716 if (IFM_OPTIONS(ifm->ifm_media) & IFM_FDX) { 1717 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1718 } else { 1719 if (IFM_OPTIONS(ifm->ifm_media) & 1720 (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) { 1721 if (bootverbose) { 1722 if_printf(ifp, "Flow control is not " 1723 "allowed for half-duplex\n"); 1724 } 1725 return EINVAL; 1726 } 1727 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1728 } 1729 adapter->hw.mac.autoneg = FALSE; 1730 adapter->hw.phy.autoneg_advertised = 0; 1731 break; 1732 1733 default: 1734 if (bootverbose) { 1735 if_printf(ifp, "Unsupported media type %d\n", 1736 IFM_SUBTYPE(ifm->ifm_media)); 1737 } 1738 return EINVAL; 1739 } 1740 adapter->ifm_flowctrl = ifm->ifm_media & IFM_ETH_FCMASK; 1741 1742 if (ifp->if_flags & IFF_RUNNING) 1743 em_init(adapter); 1744 1745 return (0); 1746 } 1747 1748 static int 1749 em_encap(struct adapter *adapter, struct mbuf **m_headp, 1750 int *segs_used, int *idx) 1751 { 1752 bus_dma_segment_t segs[EM_MAX_SCATTER]; 1753 bus_dmamap_t map; 1754 struct em_buffer *tx_buffer, *tx_buffer_mapped; 1755 struct e1000_tx_desc *ctxd = NULL; 1756 struct mbuf *m_head = *m_headp; 1757 uint32_t txd_upper, txd_lower, txd_used, cmd = 0; 1758 int maxsegs, nsegs, i, j, first, last = 0, error; 1759 1760 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1761 error = em_tso_pullup(adapter, m_headp); 1762 if (error) 1763 return error; 1764 m_head = *m_headp; 1765 } 1766 1767 txd_upper = txd_lower = 0; 1768 txd_used = 0; 1769 1770 /* 1771 * Capture the first descriptor index, this descriptor 1772 * will have the index of the EOP which is the only one 1773 * that now gets a DONE bit writeback. 1774 */ 1775 first = adapter->next_avail_tx_desc; 1776 tx_buffer = &adapter->tx_buffer_area[first]; 1777 tx_buffer_mapped = tx_buffer; 1778 map = tx_buffer->map; 1779 1780 maxsegs = adapter->num_tx_desc_avail - EM_TX_RESERVED; 1781 KASSERT(maxsegs >= adapter->spare_tx_desc, 1782 ("not enough spare TX desc")); 1783 if (adapter->pcix_82544) { 1784 /* Half it; see the comment in em_attach() */ 1785 maxsegs >>= 1; 1786 } 1787 if (maxsegs > EM_MAX_SCATTER) 1788 maxsegs = EM_MAX_SCATTER; 1789 1790 error = bus_dmamap_load_mbuf_defrag(adapter->txtag, map, m_headp, 1791 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT); 1792 if (error) { 1793 if (error == ENOBUFS) 1794 adapter->mbuf_alloc_failed++; 1795 else 1796 adapter->no_tx_dma_setup++; 1797 1798 m_freem(*m_headp); 1799 *m_headp = NULL; 1800 return error; 1801 } 1802 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); 1803 1804 m_head = *m_headp; 1805 adapter->tx_nsegs += nsegs; 1806 *segs_used += nsegs; 1807 1808 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1809 /* TSO will consume one TX desc */ 1810 i = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower); 1811 adapter->tx_nsegs += i; 1812 *segs_used += i; 1813 } else if (m_head->m_pkthdr.csum_flags & EM_CSUM_FEATURES) { 1814 /* TX csum offloading will consume one TX desc */ 1815 i = em_txcsum(adapter, m_head, &txd_upper, &txd_lower); 1816 adapter->tx_nsegs += i; 1817 *segs_used += i; 1818 } 1819 1820 /* Handle VLAN tag */ 1821 if (m_head->m_flags & M_VLANTAG) { 1822 /* Set the vlan id. */ 1823 txd_upper |= (htole16(m_head->m_pkthdr.ether_vlantag) << 16); 1824 /* Tell hardware to add tag */ 1825 txd_lower |= htole32(E1000_TXD_CMD_VLE); 1826 } 1827 1828 i = adapter->next_avail_tx_desc; 1829 1830 /* Set up our transmit descriptors */ 1831 for (j = 0; j < nsegs; j++) { 1832 /* If adapter is 82544 and on PCIX bus */ 1833 if(adapter->pcix_82544) { 1834 DESC_ARRAY desc_array; 1835 uint32_t array_elements, counter; 1836 1837 /* 1838 * Check the Address and Length combination and 1839 * split the data accordingly 1840 */ 1841 array_elements = em_82544_fill_desc(segs[j].ds_addr, 1842 segs[j].ds_len, &desc_array); 1843 for (counter = 0; counter < array_elements; counter++) { 1844 KKASSERT(txd_used < adapter->num_tx_desc_avail); 1845 1846 tx_buffer = &adapter->tx_buffer_area[i]; 1847 ctxd = &adapter->tx_desc_base[i]; 1848 1849 ctxd->buffer_addr = htole64( 1850 desc_array.descriptor[counter].address); 1851 ctxd->lower.data = htole32( 1852 E1000_TXD_CMD_IFCS | txd_lower | 1853 desc_array.descriptor[counter].length); 1854 ctxd->upper.data = htole32(txd_upper); 1855 1856 last = i; 1857 if (++i == adapter->num_tx_desc) 1858 i = 0; 1859 1860 txd_used++; 1861 } 1862 } else { 1863 tx_buffer = &adapter->tx_buffer_area[i]; 1864 ctxd = &adapter->tx_desc_base[i]; 1865 1866 ctxd->buffer_addr = htole64(segs[j].ds_addr); 1867 ctxd->lower.data = htole32(E1000_TXD_CMD_IFCS | 1868 txd_lower | segs[j].ds_len); 1869 ctxd->upper.data = htole32(txd_upper); 1870 1871 last = i; 1872 if (++i == adapter->num_tx_desc) 1873 i = 0; 1874 } 1875 } 1876 1877 adapter->next_avail_tx_desc = i; 1878 if (adapter->pcix_82544) { 1879 KKASSERT(adapter->num_tx_desc_avail > txd_used); 1880 adapter->num_tx_desc_avail -= txd_used; 1881 } else { 1882 KKASSERT(adapter->num_tx_desc_avail > nsegs); 1883 adapter->num_tx_desc_avail -= nsegs; 1884 } 1885 adapter->tx_nmbuf++; 1886 1887 tx_buffer->m_head = m_head; 1888 tx_buffer_mapped->map = tx_buffer->map; 1889 tx_buffer->map = map; 1890 1891 if (adapter->tx_nsegs >= adapter->tx_int_nsegs) { 1892 adapter->tx_nsegs = 0; 1893 1894 /* 1895 * Report Status (RS) is turned on 1896 * every tx_int_nsegs descriptors. 1897 */ 1898 cmd = E1000_TXD_CMD_RS; 1899 1900 /* 1901 * Keep track of the descriptor, which will 1902 * be written back by hardware. 1903 */ 1904 adapter->tx_dd[adapter->tx_dd_tail] = last; 1905 EM_INC_TXDD_IDX(adapter->tx_dd_tail); 1906 KKASSERT(adapter->tx_dd_tail != adapter->tx_dd_head); 1907 } 1908 1909 /* 1910 * Last Descriptor of Packet needs End Of Packet (EOP) 1911 */ 1912 ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | cmd); 1913 1914 if (adapter->hw.mac.type == e1000_82547) { 1915 /* 1916 * Advance the Transmit Descriptor Tail (TDT), this tells the 1917 * E1000 that this frame is available to transmit. 1918 */ 1919 if (adapter->link_duplex == HALF_DUPLEX) { 1920 em_82547_move_tail_serialized(adapter); 1921 } else { 1922 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); 1923 em_82547_update_fifo_head(adapter, 1924 m_head->m_pkthdr.len); 1925 } 1926 } else { 1927 /* 1928 * Defer TDT updating, until enough descriptors are setup 1929 */ 1930 *idx = i; 1931 } 1932 return (0); 1933 } 1934 1935 /* 1936 * 82547 workaround to avoid controller hang in half-duplex environment. 1937 * The workaround is to avoid queuing a large packet that would span 1938 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers 1939 * in this case. We do that only when FIFO is quiescent. 1940 */ 1941 static void 1942 em_82547_move_tail_serialized(struct adapter *adapter) 1943 { 1944 struct e1000_tx_desc *tx_desc; 1945 uint16_t hw_tdt, sw_tdt, length = 0; 1946 bool eop = 0; 1947 1948 ASSERT_SERIALIZED(adapter->arpcom.ac_if.if_serializer); 1949 1950 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); 1951 sw_tdt = adapter->next_avail_tx_desc; 1952 1953 while (hw_tdt != sw_tdt) { 1954 tx_desc = &adapter->tx_desc_base[hw_tdt]; 1955 length += tx_desc->lower.flags.length; 1956 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; 1957 if (++hw_tdt == adapter->num_tx_desc) 1958 hw_tdt = 0; 1959 1960 if (eop) { 1961 if (em_82547_fifo_workaround(adapter, length)) { 1962 adapter->tx_fifo_wrk_cnt++; 1963 callout_reset(&adapter->tx_fifo_timer, 1, 1964 em_82547_move_tail, adapter); 1965 break; 1966 } 1967 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); 1968 em_82547_update_fifo_head(adapter, length); 1969 length = 0; 1970 } 1971 } 1972 } 1973 1974 static void 1975 em_82547_move_tail(void *xsc) 1976 { 1977 struct adapter *adapter = xsc; 1978 struct ifnet *ifp = &adapter->arpcom.ac_if; 1979 1980 lwkt_serialize_enter(ifp->if_serializer); 1981 em_82547_move_tail_serialized(adapter); 1982 lwkt_serialize_exit(ifp->if_serializer); 1983 } 1984 1985 static int 1986 em_82547_fifo_workaround(struct adapter *adapter, int len) 1987 { 1988 int fifo_space, fifo_pkt_len; 1989 1990 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 1991 1992 if (adapter->link_duplex == HALF_DUPLEX) { 1993 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; 1994 1995 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { 1996 if (em_82547_tx_fifo_reset(adapter)) 1997 return (0); 1998 else 1999 return (1); 2000 } 2001 } 2002 return (0); 2003 } 2004 2005 static void 2006 em_82547_update_fifo_head(struct adapter *adapter, int len) 2007 { 2008 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2009 2010 /* tx_fifo_head is always 16 byte aligned */ 2011 adapter->tx_fifo_head += fifo_pkt_len; 2012 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) 2013 adapter->tx_fifo_head -= adapter->tx_fifo_size; 2014 } 2015 2016 static int 2017 em_82547_tx_fifo_reset(struct adapter *adapter) 2018 { 2019 uint32_t tctl; 2020 2021 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == 2022 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && 2023 (E1000_READ_REG(&adapter->hw, E1000_TDFT) == 2024 E1000_READ_REG(&adapter->hw, E1000_TDFH)) && 2025 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == 2026 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && 2027 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { 2028 /* Disable TX unit */ 2029 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2030 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, 2031 tctl & ~E1000_TCTL_EN); 2032 2033 /* Reset FIFO pointers */ 2034 E1000_WRITE_REG(&adapter->hw, E1000_TDFT, 2035 adapter->tx_head_addr); 2036 E1000_WRITE_REG(&adapter->hw, E1000_TDFH, 2037 adapter->tx_head_addr); 2038 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, 2039 adapter->tx_head_addr); 2040 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, 2041 adapter->tx_head_addr); 2042 2043 /* Re-enable TX unit */ 2044 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2045 E1000_WRITE_FLUSH(&adapter->hw); 2046 2047 adapter->tx_fifo_head = 0; 2048 adapter->tx_fifo_reset_cnt++; 2049 2050 return (TRUE); 2051 } else { 2052 return (FALSE); 2053 } 2054 } 2055 2056 static void 2057 em_set_promisc(struct adapter *adapter) 2058 { 2059 struct ifnet *ifp = &adapter->arpcom.ac_if; 2060 uint32_t reg_rctl; 2061 2062 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2063 2064 if (ifp->if_flags & IFF_PROMISC) { 2065 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2066 /* Turn this on if you want to see bad packets */ 2067 if (em_debug_sbp) 2068 reg_rctl |= E1000_RCTL_SBP; 2069 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2070 } else if (ifp->if_flags & IFF_ALLMULTI) { 2071 reg_rctl |= E1000_RCTL_MPE; 2072 reg_rctl &= ~E1000_RCTL_UPE; 2073 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2074 } 2075 } 2076 2077 static void 2078 em_disable_promisc(struct adapter *adapter) 2079 { 2080 uint32_t reg_rctl; 2081 2082 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2083 2084 reg_rctl &= ~E1000_RCTL_UPE; 2085 reg_rctl &= ~E1000_RCTL_MPE; 2086 reg_rctl &= ~E1000_RCTL_SBP; 2087 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2088 } 2089 2090 static void 2091 em_set_multi(struct adapter *adapter) 2092 { 2093 struct ifnet *ifp = &adapter->arpcom.ac_if; 2094 struct ifmultiaddr *ifma; 2095 uint32_t reg_rctl = 0; 2096 uint8_t *mta; 2097 int mcnt = 0; 2098 2099 mta = adapter->mta; 2100 bzero(mta, ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); 2101 2102 if (adapter->hw.mac.type == e1000_82542 && 2103 adapter->hw.revision_id == E1000_REVISION_2) { 2104 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2105 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2106 e1000_pci_clear_mwi(&adapter->hw); 2107 reg_rctl |= E1000_RCTL_RST; 2108 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2109 msec_delay(5); 2110 } 2111 2112 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2113 if (ifma->ifma_addr->sa_family != AF_LINK) 2114 continue; 2115 2116 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2117 break; 2118 2119 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2120 &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN); 2121 mcnt++; 2122 } 2123 2124 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2125 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2126 reg_rctl |= E1000_RCTL_MPE; 2127 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2128 } else { 2129 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2130 } 2131 2132 if (adapter->hw.mac.type == e1000_82542 && 2133 adapter->hw.revision_id == E1000_REVISION_2) { 2134 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2135 reg_rctl &= ~E1000_RCTL_RST; 2136 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2137 msec_delay(5); 2138 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2139 e1000_pci_set_mwi(&adapter->hw); 2140 } 2141 } 2142 2143 /* 2144 * This routine checks for link status and updates statistics. 2145 */ 2146 static void 2147 em_timer(void *xsc) 2148 { 2149 struct adapter *adapter = xsc; 2150 struct ifnet *ifp = &adapter->arpcom.ac_if; 2151 2152 lwkt_serialize_enter(ifp->if_serializer); 2153 2154 em_update_link_status(adapter); 2155 em_update_stats(adapter); 2156 2157 /* Reset LAA into RAR[0] on 82571 */ 2158 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2159 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2160 2161 if (em_display_debug_stats && (ifp->if_flags & IFF_RUNNING)) 2162 em_print_hw_stats(adapter); 2163 2164 em_smartspeed(adapter); 2165 2166 callout_reset(&adapter->timer, hz, em_timer, adapter); 2167 2168 lwkt_serialize_exit(ifp->if_serializer); 2169 } 2170 2171 static void 2172 em_update_link_status(struct adapter *adapter) 2173 { 2174 struct e1000_hw *hw = &adapter->hw; 2175 struct ifnet *ifp = &adapter->arpcom.ac_if; 2176 device_t dev = adapter->dev; 2177 uint32_t link_check = 0; 2178 2179 /* Get the cached link value or read phy for real */ 2180 switch (hw->phy.media_type) { 2181 case e1000_media_type_copper: 2182 if (hw->mac.get_link_status) { 2183 /* Do the work to read phy */ 2184 e1000_check_for_link(hw); 2185 link_check = !hw->mac.get_link_status; 2186 if (link_check) /* ESB2 fix */ 2187 e1000_cfg_on_link_up(hw); 2188 } else { 2189 link_check = TRUE; 2190 } 2191 break; 2192 2193 case e1000_media_type_fiber: 2194 e1000_check_for_link(hw); 2195 link_check = 2196 E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU; 2197 break; 2198 2199 case e1000_media_type_internal_serdes: 2200 e1000_check_for_link(hw); 2201 link_check = adapter->hw.mac.serdes_has_link; 2202 break; 2203 2204 case e1000_media_type_unknown: 2205 default: 2206 break; 2207 } 2208 2209 /* Now check for a transition */ 2210 if (link_check && adapter->link_active == 0) { 2211 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2212 &adapter->link_duplex); 2213 2214 /* 2215 * Check if we should enable/disable SPEED_MODE bit on 2216 * 82571/82572 2217 */ 2218 if (adapter->link_speed != SPEED_1000 && 2219 (hw->mac.type == e1000_82571 || 2220 hw->mac.type == e1000_82572)) { 2221 int tarc0; 2222 2223 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2224 tarc0 &= ~SPEED_MODE_BIT; 2225 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2226 } 2227 if (bootverbose) { 2228 char flowctrl[IFM_ETH_FC_STRLEN]; 2229 2230 e1000_fc2str(hw->fc.current_mode, flowctrl, 2231 sizeof(flowctrl)); 2232 device_printf(dev, "Link is up %d Mbps %s, " 2233 "Flow control: %s\n", 2234 adapter->link_speed, 2235 (adapter->link_duplex == FULL_DUPLEX) ? 2236 "Full Duplex" : "Half Duplex", 2237 flowctrl); 2238 } 2239 if (adapter->ifm_flowctrl & IFM_ETH_FORCEPAUSE) 2240 e1000_force_flowctrl(hw, adapter->ifm_flowctrl); 2241 adapter->link_active = 1; 2242 adapter->smartspeed = 0; 2243 ifp->if_baudrate = adapter->link_speed * 1000000; 2244 ifp->if_link_state = LINK_STATE_UP; 2245 if_link_state_change(ifp); 2246 } else if (!link_check && adapter->link_active == 1) { 2247 ifp->if_baudrate = adapter->link_speed = 0; 2248 adapter->link_duplex = 0; 2249 if (bootverbose) 2250 device_printf(dev, "Link is Down\n"); 2251 adapter->link_active = 0; 2252 #if 0 2253 /* Link down, disable watchdog */ 2254 if->if_timer = 0; 2255 #endif 2256 ifp->if_link_state = LINK_STATE_DOWN; 2257 if_link_state_change(ifp); 2258 } 2259 } 2260 2261 static void 2262 em_stop(struct adapter *adapter) 2263 { 2264 struct ifnet *ifp = &adapter->arpcom.ac_if; 2265 int i; 2266 2267 ASSERT_SERIALIZED(ifp->if_serializer); 2268 2269 em_disable_intr(adapter); 2270 2271 callout_stop(&adapter->timer); 2272 callout_stop(&adapter->tx_fifo_timer); 2273 2274 ifp->if_flags &= ~IFF_RUNNING; 2275 ifq_clr_oactive(&ifp->if_snd); 2276 ifp->if_timer = 0; 2277 adapter->tx_running = 0; 2278 callout_stop(&adapter->tx_gc_timer); 2279 2280 e1000_reset_hw(&adapter->hw); 2281 if (adapter->hw.mac.type >= e1000_82544) 2282 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2283 2284 for (i = 0; i < adapter->num_tx_desc; i++) { 2285 struct em_buffer *tx_buffer = &adapter->tx_buffer_area[i]; 2286 2287 if (tx_buffer->m_head != NULL) 2288 em_free_txbuffer(adapter, tx_buffer); 2289 } 2290 2291 for (i = 0; i < adapter->num_rx_desc; i++) { 2292 struct em_buffer *rx_buffer = &adapter->rx_buffer_area[i]; 2293 2294 if (rx_buffer->m_head != NULL) { 2295 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 2296 m_freem(rx_buffer->m_head); 2297 rx_buffer->m_head = NULL; 2298 } 2299 } 2300 2301 if (adapter->fmp != NULL) 2302 m_freem(adapter->fmp); 2303 adapter->fmp = NULL; 2304 adapter->lmp = NULL; 2305 2306 adapter->csum_flags = 0; 2307 adapter->csum_lhlen = 0; 2308 adapter->csum_iphlen = 0; 2309 adapter->csum_thlen = 0; 2310 adapter->csum_mss = 0; 2311 adapter->csum_pktlen = 0; 2312 2313 adapter->tx_dd_head = 0; 2314 adapter->tx_dd_tail = 0; 2315 adapter->tx_nsegs = 0; 2316 } 2317 2318 static int 2319 em_get_hw_info(struct adapter *adapter) 2320 { 2321 device_t dev = adapter->dev; 2322 2323 /* Save off the information about this board */ 2324 adapter->hw.vendor_id = pci_get_vendor(dev); 2325 adapter->hw.device_id = pci_get_device(dev); 2326 adapter->hw.revision_id = pci_get_revid(dev); 2327 adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev); 2328 adapter->hw.subsystem_device_id = pci_get_subdevice(dev); 2329 2330 /* Do Shared Code Init and Setup */ 2331 if (e1000_set_mac_type(&adapter->hw)) 2332 return ENXIO; 2333 return 0; 2334 } 2335 2336 static int 2337 em_alloc_pci_res(struct adapter *adapter) 2338 { 2339 device_t dev = adapter->dev; 2340 u_int intr_flags; 2341 int val, rid, msi_enable, cap; 2342 2343 /* Enable bus mastering */ 2344 pci_enable_busmaster(dev); 2345 2346 adapter->memory_rid = EM_BAR_MEM; 2347 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2348 &adapter->memory_rid, RF_ACTIVE); 2349 if (adapter->memory == NULL) { 2350 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2351 return (ENXIO); 2352 } 2353 adapter->osdep.mem_bus_space_tag = 2354 rman_get_bustag(adapter->memory); 2355 adapter->osdep.mem_bus_space_handle = 2356 rman_get_bushandle(adapter->memory); 2357 2358 /* XXX This is quite goofy, it is not actually used */ 2359 adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle; 2360 2361 /* Only older adapters use IO mapping */ 2362 if (adapter->hw.mac.type > e1000_82543 && 2363 adapter->hw.mac.type < e1000_82571) { 2364 /* Figure our where our IO BAR is ? */ 2365 for (rid = PCIR_BAR(0); rid < PCIR_CARDBUSCIS;) { 2366 val = pci_read_config(dev, rid, 4); 2367 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 2368 adapter->io_rid = rid; 2369 break; 2370 } 2371 rid += 4; 2372 /* check for 64bit BAR */ 2373 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 2374 rid += 4; 2375 } 2376 if (rid >= PCIR_CARDBUSCIS) { 2377 device_printf(dev, "Unable to locate IO BAR\n"); 2378 return (ENXIO); 2379 } 2380 adapter->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT, 2381 &adapter->io_rid, RF_ACTIVE); 2382 if (adapter->ioport == NULL) { 2383 device_printf(dev, "Unable to allocate bus resource: " 2384 "ioport\n"); 2385 return (ENXIO); 2386 } 2387 adapter->hw.io_base = 0; 2388 adapter->osdep.io_bus_space_tag = 2389 rman_get_bustag(adapter->ioport); 2390 adapter->osdep.io_bus_space_handle = 2391 rman_get_bushandle(adapter->ioport); 2392 } 2393 2394 /* 2395 * Don't enable MSI-X on 82574, see: 2396 * 82574 specification update errata #15 2397 * 2398 * Don't enable MSI on PCI/PCI-X chips, see: 2399 * 82540 specification update errata #6 2400 * 82545 specification update errata #4 2401 * 2402 * Don't enable MSI on 82571/82572, see: 2403 * 82571/82572 specification update errata #63 2404 * 2405 * Some versions of I219 only have PCI AF. 2406 */ 2407 msi_enable = em_msi_enable; 2408 if (msi_enable && 2409 (!(pci_is_pcie(dev) || 2410 pci_find_extcap(dev, PCIY_PCIAF, &cap) == 0) || 2411 adapter->hw.mac.type == e1000_82571 || 2412 adapter->hw.mac.type == e1000_82572)) 2413 msi_enable = 0; 2414 again: 2415 adapter->intr_type = pci_alloc_1intr(dev, msi_enable, 2416 &adapter->intr_rid, &intr_flags); 2417 2418 if (adapter->intr_type == PCI_INTR_TYPE_LEGACY) { 2419 int unshared; 2420 2421 unshared = device_getenv_int(dev, "irq.unshared", 0); 2422 if (!unshared) { 2423 adapter->flags |= EM_FLAG_SHARED_INTR; 2424 if (bootverbose) 2425 device_printf(dev, "IRQ shared\n"); 2426 } else { 2427 intr_flags &= ~RF_SHAREABLE; 2428 if (bootverbose) 2429 device_printf(dev, "IRQ unshared\n"); 2430 } 2431 } 2432 2433 adapter->intr_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 2434 &adapter->intr_rid, intr_flags); 2435 if (adapter->intr_res == NULL) { 2436 device_printf(dev, "Unable to allocate bus resource: %s\n", 2437 adapter->intr_type == PCI_INTR_TYPE_MSI ? 2438 "MSI" : "legacy intr"); 2439 if (!msi_enable) { 2440 /* Retry with MSI. */ 2441 msi_enable = 1; 2442 adapter->flags &= ~EM_FLAG_SHARED_INTR; 2443 goto again; 2444 } 2445 return (ENXIO); 2446 } 2447 2448 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2449 adapter->hw.back = &adapter->osdep; 2450 return (0); 2451 } 2452 2453 static void 2454 em_free_pci_res(struct adapter *adapter) 2455 { 2456 device_t dev = adapter->dev; 2457 2458 if (adapter->intr_res != NULL) { 2459 bus_release_resource(dev, SYS_RES_IRQ, 2460 adapter->intr_rid, adapter->intr_res); 2461 } 2462 2463 if (adapter->intr_type == PCI_INTR_TYPE_MSI) 2464 pci_release_msi(dev); 2465 2466 if (adapter->memory != NULL) { 2467 bus_release_resource(dev, SYS_RES_MEMORY, 2468 adapter->memory_rid, adapter->memory); 2469 } 2470 2471 if (adapter->flash != NULL) { 2472 bus_release_resource(dev, SYS_RES_MEMORY, 2473 adapter->flash_rid, adapter->flash); 2474 } 2475 2476 if (adapter->ioport != NULL) { 2477 bus_release_resource(dev, SYS_RES_IOPORT, 2478 adapter->io_rid, adapter->ioport); 2479 } 2480 } 2481 2482 static int 2483 em_reset(struct adapter *adapter) 2484 { 2485 device_t dev = adapter->dev; 2486 uint16_t rx_buffer_size; 2487 uint32_t pba; 2488 2489 /* When hardware is reset, fifo_head is also reset */ 2490 adapter->tx_fifo_head = 0; 2491 2492 /* Set up smart power down as default off on newer adapters. */ 2493 if (!em_smart_pwr_down && 2494 (adapter->hw.mac.type == e1000_82571 || 2495 adapter->hw.mac.type == e1000_82572)) { 2496 uint16_t phy_tmp = 0; 2497 2498 /* Speed up time to link by disabling smart power down. */ 2499 e1000_read_phy_reg(&adapter->hw, 2500 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 2501 phy_tmp &= ~IGP02E1000_PM_SPD; 2502 e1000_write_phy_reg(&adapter->hw, 2503 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 2504 } 2505 2506 /* 2507 * Packet Buffer Allocation (PBA) 2508 * Writing PBA sets the receive portion of the buffer 2509 * the remainder is used for the transmit buffer. 2510 * 2511 * Devices before the 82547 had a Packet Buffer of 64K. 2512 * Default allocation: PBA=48K for Rx, leaving 16K for Tx. 2513 * After the 82547 the buffer was reduced to 40K. 2514 * Default allocation: PBA=30K for Rx, leaving 10K for Tx. 2515 * Note: default does not leave enough room for Jumbo Frame >10k. 2516 */ 2517 switch (adapter->hw.mac.type) { 2518 case e1000_82547: 2519 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ 2520 if (adapter->hw.mac.max_frame_size > 8192) 2521 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 2522 else 2523 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 2524 adapter->tx_fifo_head = 0; 2525 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; 2526 adapter->tx_fifo_size = 2527 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; 2528 break; 2529 2530 /* Total Packet Buffer on these is 48K */ 2531 case e1000_82571: 2532 case e1000_82572: 2533 case e1000_80003es2lan: 2534 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 2535 break; 2536 2537 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 2538 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 2539 break; 2540 2541 case e1000_82574: 2542 case e1000_82583: 2543 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 2544 break; 2545 2546 case e1000_ich8lan: 2547 pba = E1000_PBA_8K; 2548 break; 2549 2550 case e1000_ich9lan: 2551 case e1000_ich10lan: 2552 #define E1000_PBA_10K 0x000A 2553 pba = E1000_PBA_10K; 2554 break; 2555 2556 case e1000_pchlan: 2557 case e1000_pch2lan: 2558 case e1000_pch_lpt: 2559 case e1000_pch_spt: 2560 pba = E1000_PBA_26K; 2561 break; 2562 2563 default: 2564 /* Devices before 82547 had a Packet Buffer of 64K. */ 2565 if (adapter->hw.mac.max_frame_size > 8192) 2566 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 2567 else 2568 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 2569 } 2570 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 2571 2572 /* 2573 * These parameters control the automatic generation (Tx) and 2574 * response (Rx) to Ethernet PAUSE frames. 2575 * - High water mark should allow for at least two frames to be 2576 * received after sending an XOFF. 2577 * - Low water mark works best when it is very near the high water mark. 2578 * This allows the receiver to restart by sending XON when it has 2579 * drained a bit. Here we use an arbitary value of 1500 which will 2580 * restart after one full frame is pulled from the buffer. There 2581 * could be several smaller frames in the buffer and if so they will 2582 * not trigger the XON until their total number reduces the buffer 2583 * by 1500. 2584 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2585 */ 2586 rx_buffer_size = 2587 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) << 10; 2588 2589 adapter->hw.fc.high_water = rx_buffer_size - 2590 roundup2(adapter->hw.mac.max_frame_size, 1024); 2591 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 2592 2593 if (adapter->hw.mac.type == e1000_80003es2lan) 2594 adapter->hw.fc.pause_time = 0xFFFF; 2595 else 2596 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; 2597 2598 adapter->hw.fc.send_xon = TRUE; 2599 2600 adapter->hw.fc.requested_mode = e1000_ifmedia2fc(adapter->ifm_flowctrl); 2601 2602 /* 2603 * Device specific overrides/settings 2604 */ 2605 switch (adapter->hw.mac.type) { 2606 case e1000_pchlan: 2607 KASSERT(adapter->hw.fc.requested_mode == e1000_fc_rx_pause || 2608 adapter->hw.fc.requested_mode == e1000_fc_none, 2609 ("unsupported flow control on PCH %d", 2610 adapter->hw.fc.requested_mode)); 2611 adapter->hw.fc.pause_time = 0xFFFF; /* override */ 2612 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) { 2613 adapter->hw.fc.high_water = 0x3500; 2614 adapter->hw.fc.low_water = 0x1500; 2615 } else { 2616 adapter->hw.fc.high_water = 0x5000; 2617 adapter->hw.fc.low_water = 0x3000; 2618 } 2619 adapter->hw.fc.refresh_time = 0x1000; 2620 break; 2621 2622 case e1000_pch2lan: 2623 case e1000_pch_lpt: 2624 case e1000_pch_spt: 2625 adapter->hw.fc.high_water = 0x5C20; 2626 adapter->hw.fc.low_water = 0x5048; 2627 adapter->hw.fc.pause_time = 0x0650; 2628 adapter->hw.fc.refresh_time = 0x0400; 2629 /* Jumbos need adjusted PBA */ 2630 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) 2631 E1000_WRITE_REG(&adapter->hw, E1000_PBA, 12); 2632 else 2633 E1000_WRITE_REG(&adapter->hw, E1000_PBA, 26); 2634 break; 2635 2636 case e1000_ich9lan: 2637 case e1000_ich10lan: 2638 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) { 2639 adapter->hw.fc.high_water = 0x2800; 2640 adapter->hw.fc.low_water = 2641 adapter->hw.fc.high_water - 8; 2642 break; 2643 } 2644 /* FALL THROUGH */ 2645 default: 2646 if (adapter->hw.mac.type == e1000_80003es2lan) 2647 adapter->hw.fc.pause_time = 0xFFFF; 2648 break; 2649 } 2650 2651 /* Issue a global reset */ 2652 e1000_reset_hw(&adapter->hw); 2653 if (adapter->hw.mac.type >= e1000_82544) 2654 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2655 em_disable_aspm(adapter); 2656 2657 if (e1000_init_hw(&adapter->hw) < 0) { 2658 device_printf(dev, "Hardware Initialization Failed\n"); 2659 return (EIO); 2660 } 2661 2662 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 2663 e1000_get_phy_info(&adapter->hw); 2664 e1000_check_for_link(&adapter->hw); 2665 2666 return (0); 2667 } 2668 2669 static void 2670 em_setup_ifp(struct adapter *adapter) 2671 { 2672 struct ifnet *ifp = &adapter->arpcom.ac_if; 2673 2674 if_initname(ifp, device_get_name(adapter->dev), 2675 device_get_unit(adapter->dev)); 2676 ifp->if_softc = adapter; 2677 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2678 ifp->if_init = em_init; 2679 ifp->if_ioctl = em_ioctl; 2680 ifp->if_start = em_start; 2681 #ifdef IFPOLL_ENABLE 2682 ifp->if_npoll = em_npoll; 2683 #endif 2684 ifp->if_watchdog = em_watchdog; 2685 ifp->if_nmbclusters = adapter->num_rx_desc; 2686 ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1); 2687 ifq_set_ready(&ifp->if_snd); 2688 2689 ether_ifattach(ifp, adapter->hw.mac.addr, NULL); 2690 2691 ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2692 if (adapter->hw.mac.type >= e1000_82543) 2693 ifp->if_capabilities |= IFCAP_HWCSUM; 2694 if (adapter->flags & EM_FLAG_TSO) 2695 ifp->if_capabilities |= IFCAP_TSO; 2696 ifp->if_capenable = ifp->if_capabilities; 2697 2698 if (ifp->if_capenable & IFCAP_TXCSUM) 2699 ifp->if_hwassist |= EM_CSUM_FEATURES; 2700 if (ifp->if_capenable & IFCAP_TSO) 2701 ifp->if_hwassist |= CSUM_TSO; 2702 2703 /* 2704 * Tell the upper layer(s) we support long frames. 2705 */ 2706 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2707 2708 /* 2709 * Specify the media types supported by this adapter and register 2710 * callbacks to update media and link information 2711 */ 2712 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 2713 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 2714 u_char fiber_type = IFM_1000_SX; /* default type */ 2715 2716 if (adapter->hw.mac.type == e1000_82545) 2717 fiber_type = IFM_1000_LX; 2718 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 2719 0, NULL); 2720 } else { 2721 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2722 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2723 0, NULL); 2724 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2725 0, NULL); 2726 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2727 0, NULL); 2728 if (adapter->hw.phy.type != e1000_phy_ife) { 2729 ifmedia_add(&adapter->media, 2730 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2731 } 2732 } 2733 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2734 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO | 2735 adapter->ifm_flowctrl); 2736 } 2737 2738 2739 /* 2740 * Workaround for SmartSpeed on 82541 and 82547 controllers 2741 */ 2742 static void 2743 em_smartspeed(struct adapter *adapter) 2744 { 2745 uint16_t phy_tmp; 2746 2747 if (adapter->link_active || adapter->hw.phy.type != e1000_phy_igp || 2748 adapter->hw.mac.autoneg == 0 || 2749 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 2750 return; 2751 2752 if (adapter->smartspeed == 0) { 2753 /* 2754 * If Master/Slave config fault is asserted twice, 2755 * we assume back-to-back 2756 */ 2757 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 2758 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 2759 return; 2760 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 2761 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 2762 e1000_read_phy_reg(&adapter->hw, 2763 PHY_1000T_CTRL, &phy_tmp); 2764 if (phy_tmp & CR_1000T_MS_ENABLE) { 2765 phy_tmp &= ~CR_1000T_MS_ENABLE; 2766 e1000_write_phy_reg(&adapter->hw, 2767 PHY_1000T_CTRL, phy_tmp); 2768 adapter->smartspeed++; 2769 if (adapter->hw.mac.autoneg && 2770 !e1000_phy_setup_autoneg(&adapter->hw) && 2771 !e1000_read_phy_reg(&adapter->hw, 2772 PHY_CONTROL, &phy_tmp)) { 2773 phy_tmp |= MII_CR_AUTO_NEG_EN | 2774 MII_CR_RESTART_AUTO_NEG; 2775 e1000_write_phy_reg(&adapter->hw, 2776 PHY_CONTROL, phy_tmp); 2777 } 2778 } 2779 } 2780 return; 2781 } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 2782 /* If still no link, perhaps using 2/3 pair cable */ 2783 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); 2784 phy_tmp |= CR_1000T_MS_ENABLE; 2785 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); 2786 if (adapter->hw.mac.autoneg && 2787 !e1000_phy_setup_autoneg(&adapter->hw) && 2788 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { 2789 phy_tmp |= MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG; 2790 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); 2791 } 2792 } 2793 2794 /* Restart process after EM_SMARTSPEED_MAX iterations */ 2795 if (adapter->smartspeed++ == EM_SMARTSPEED_MAX) 2796 adapter->smartspeed = 0; 2797 } 2798 2799 static int 2800 em_dma_malloc(struct adapter *adapter, bus_size_t size, 2801 struct em_dma_alloc *dma) 2802 { 2803 dma->dma_vaddr = bus_dmamem_coherent_any(adapter->parent_dtag, 2804 EM_DBA_ALIGN, size, BUS_DMA_WAITOK, 2805 &dma->dma_tag, &dma->dma_map, 2806 &dma->dma_paddr); 2807 if (dma->dma_vaddr == NULL) 2808 return ENOMEM; 2809 else 2810 return 0; 2811 } 2812 2813 static void 2814 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 2815 { 2816 if (dma->dma_tag == NULL) 2817 return; 2818 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2819 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2820 bus_dma_tag_destroy(dma->dma_tag); 2821 } 2822 2823 static int 2824 em_create_tx_ring(struct adapter *adapter) 2825 { 2826 device_t dev = adapter->dev; 2827 struct em_buffer *tx_buffer; 2828 int error, i; 2829 2830 adapter->tx_buffer_area = 2831 kmalloc(sizeof(struct em_buffer) * adapter->num_tx_desc, 2832 M_DEVBUF, M_WAITOK | M_ZERO); 2833 2834 /* 2835 * Create DMA tags for tx buffers 2836 */ 2837 error = bus_dma_tag_create(adapter->parent_dtag, /* parent */ 2838 1, 0, /* alignment, bounds */ 2839 BUS_SPACE_MAXADDR, /* lowaddr */ 2840 BUS_SPACE_MAXADDR, /* highaddr */ 2841 NULL, NULL, /* filter, filterarg */ 2842 EM_TSO_SIZE, /* maxsize */ 2843 EM_MAX_SCATTER, /* nsegments */ 2844 PAGE_SIZE, /* maxsegsize */ 2845 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | 2846 BUS_DMA_ONEBPAGE, /* flags */ 2847 &adapter->txtag); 2848 if (error) { 2849 device_printf(dev, "Unable to allocate TX DMA tag\n"); 2850 kfree(adapter->tx_buffer_area, M_DEVBUF); 2851 adapter->tx_buffer_area = NULL; 2852 return error; 2853 } 2854 2855 /* 2856 * Create DMA maps for tx buffers 2857 */ 2858 for (i = 0; i < adapter->num_tx_desc; i++) { 2859 tx_buffer = &adapter->tx_buffer_area[i]; 2860 2861 error = bus_dmamap_create(adapter->txtag, 2862 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE, 2863 &tx_buffer->map); 2864 if (error) { 2865 device_printf(dev, "Unable to create TX DMA map\n"); 2866 em_destroy_tx_ring(adapter, i); 2867 return error; 2868 } 2869 } 2870 return (0); 2871 } 2872 2873 static void 2874 em_init_tx_ring(struct adapter *adapter) 2875 { 2876 /* Clear the old ring contents */ 2877 bzero(adapter->tx_desc_base, 2878 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 2879 2880 /* Reset state */ 2881 adapter->next_avail_tx_desc = 0; 2882 adapter->next_tx_to_clean = 0; 2883 adapter->num_tx_desc_avail = adapter->num_tx_desc; 2884 adapter->tx_nmbuf = 0; 2885 adapter->tx_running = 0; 2886 } 2887 2888 static void 2889 em_init_tx_unit(struct adapter *adapter) 2890 { 2891 uint32_t tctl, tarc, tipg = 0; 2892 uint64_t bus_addr; 2893 2894 /* Setup the Base and Length of the Tx Descriptor Ring */ 2895 bus_addr = adapter->txdma.dma_paddr; 2896 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), 2897 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 2898 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), 2899 (uint32_t)(bus_addr >> 32)); 2900 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), 2901 (uint32_t)bus_addr); 2902 /* Setup the HW Tx Head and Tail descriptor pointers */ 2903 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); 2904 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); 2905 2906 /* Set the default values for the Tx Inter Packet Gap timer */ 2907 switch (adapter->hw.mac.type) { 2908 case e1000_82542: 2909 tipg = DEFAULT_82542_TIPG_IPGT; 2910 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 2911 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 2912 break; 2913 2914 case e1000_80003es2lan: 2915 tipg = DEFAULT_82543_TIPG_IPGR1; 2916 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 2917 E1000_TIPG_IPGR2_SHIFT; 2918 break; 2919 2920 default: 2921 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 2922 adapter->hw.phy.media_type == 2923 e1000_media_type_internal_serdes) 2924 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 2925 else 2926 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 2927 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 2928 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 2929 break; 2930 } 2931 2932 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 2933 2934 /* NOTE: 0 is not allowed for TIDV */ 2935 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, 1); 2936 if(adapter->hw.mac.type >= e1000_82540) 2937 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 0); 2938 2939 if (adapter->hw.mac.type == e1000_82571 || 2940 adapter->hw.mac.type == e1000_82572) { 2941 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 2942 tarc |= SPEED_MODE_BIT; 2943 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 2944 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 2945 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 2946 tarc |= 1; 2947 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 2948 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 2949 tarc |= 1; 2950 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 2951 } 2952 2953 /* Program the Transmit Control Register */ 2954 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2955 tctl &= ~E1000_TCTL_CT; 2956 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 2957 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2958 2959 if (adapter->hw.mac.type >= e1000_82571) 2960 tctl |= E1000_TCTL_MULR; 2961 2962 /* This write will effectively turn on the transmit unit. */ 2963 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2964 2965 if (adapter->hw.mac.type == e1000_82571 || 2966 adapter->hw.mac.type == e1000_82572 || 2967 adapter->hw.mac.type == e1000_80003es2lan) { 2968 /* Bit 28 of TARC1 must be cleared when MULR is enabled */ 2969 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 2970 tarc &= ~(1 << 28); 2971 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 2972 } 2973 } 2974 2975 static void 2976 em_destroy_tx_ring(struct adapter *adapter, int ndesc) 2977 { 2978 struct em_buffer *tx_buffer; 2979 int i; 2980 2981 if (adapter->tx_buffer_area == NULL) 2982 return; 2983 2984 for (i = 0; i < ndesc; i++) { 2985 tx_buffer = &adapter->tx_buffer_area[i]; 2986 2987 KKASSERT(tx_buffer->m_head == NULL); 2988 bus_dmamap_destroy(adapter->txtag, tx_buffer->map); 2989 } 2990 bus_dma_tag_destroy(adapter->txtag); 2991 2992 kfree(adapter->tx_buffer_area, M_DEVBUF); 2993 adapter->tx_buffer_area = NULL; 2994 } 2995 2996 /* 2997 * The offload context needs to be set when we transfer the first 2998 * packet of a particular protocol (TCP/UDP). This routine has been 2999 * enhanced to deal with inserted VLAN headers. 3000 * 3001 * If the new packet's ether header length, ip header length and 3002 * csum offloading type are same as the previous packet, we should 3003 * avoid allocating a new csum context descriptor; mainly to take 3004 * advantage of the pipeline effect of the TX data read request. 3005 * 3006 * This function returns number of TX descrptors allocated for 3007 * csum context. 3008 */ 3009 static int 3010 em_txcsum(struct adapter *adapter, struct mbuf *mp, 3011 uint32_t *txd_upper, uint32_t *txd_lower) 3012 { 3013 struct e1000_context_desc *TXD; 3014 int curr_txd, ehdrlen, csum_flags; 3015 uint32_t cmd, hdr_len, ip_hlen; 3016 3017 csum_flags = mp->m_pkthdr.csum_flags & EM_CSUM_FEATURES; 3018 ip_hlen = mp->m_pkthdr.csum_iphlen; 3019 ehdrlen = mp->m_pkthdr.csum_lhlen; 3020 3021 if (adapter->csum_lhlen == ehdrlen && 3022 adapter->csum_iphlen == ip_hlen && 3023 adapter->csum_flags == csum_flags) { 3024 /* 3025 * Same csum offload context as the previous packets; 3026 * just return. 3027 */ 3028 *txd_upper = adapter->csum_txd_upper; 3029 *txd_lower = adapter->csum_txd_lower; 3030 return 0; 3031 } 3032 3033 /* 3034 * Setup a new csum offload context. 3035 */ 3036 3037 curr_txd = adapter->next_avail_tx_desc; 3038 TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd]; 3039 3040 cmd = 0; 3041 3042 /* Setup of IP header checksum. */ 3043 if (csum_flags & CSUM_IP) { 3044 /* 3045 * Start offset for header checksum calculation. 3046 * End offset for header checksum calculation. 3047 * Offset of place to put the checksum. 3048 */ 3049 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3050 TXD->lower_setup.ip_fields.ipcse = 3051 htole16(ehdrlen + ip_hlen - 1); 3052 TXD->lower_setup.ip_fields.ipcso = 3053 ehdrlen + offsetof(struct ip, ip_sum); 3054 cmd |= E1000_TXD_CMD_IP; 3055 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3056 } 3057 hdr_len = ehdrlen + ip_hlen; 3058 3059 if (csum_flags & CSUM_TCP) { 3060 /* 3061 * Start offset for payload checksum calculation. 3062 * End offset for payload checksum calculation. 3063 * Offset of place to put the checksum. 3064 */ 3065 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3066 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3067 TXD->upper_setup.tcp_fields.tucso = 3068 hdr_len + offsetof(struct tcphdr, th_sum); 3069 cmd |= E1000_TXD_CMD_TCP; 3070 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3071 } else if (csum_flags & CSUM_UDP) { 3072 /* 3073 * Start offset for header checksum calculation. 3074 * End offset for header checksum calculation. 3075 * Offset of place to put the checksum. 3076 */ 3077 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3078 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3079 TXD->upper_setup.tcp_fields.tucso = 3080 hdr_len + offsetof(struct udphdr, uh_sum); 3081 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3082 } 3083 3084 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 3085 E1000_TXD_DTYP_D; /* Data descr */ 3086 3087 /* Save the information for this csum offloading context */ 3088 adapter->csum_lhlen = ehdrlen; 3089 adapter->csum_iphlen = ip_hlen; 3090 adapter->csum_flags = csum_flags; 3091 adapter->csum_txd_upper = *txd_upper; 3092 adapter->csum_txd_lower = *txd_lower; 3093 3094 TXD->tcp_seg_setup.data = htole32(0); 3095 TXD->cmd_and_length = 3096 htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 3097 3098 if (++curr_txd == adapter->num_tx_desc) 3099 curr_txd = 0; 3100 3101 KKASSERT(adapter->num_tx_desc_avail > 0); 3102 adapter->num_tx_desc_avail--; 3103 3104 adapter->next_avail_tx_desc = curr_txd; 3105 return 1; 3106 } 3107 3108 static void 3109 em_txeof(struct adapter *adapter) 3110 { 3111 struct ifnet *ifp = &adapter->arpcom.ac_if; 3112 struct em_buffer *tx_buffer; 3113 int first, num_avail; 3114 3115 if (adapter->tx_dd_head == adapter->tx_dd_tail) 3116 return; 3117 3118 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 3119 return; 3120 3121 num_avail = adapter->num_tx_desc_avail; 3122 first = adapter->next_tx_to_clean; 3123 3124 while (adapter->tx_dd_head != adapter->tx_dd_tail) { 3125 struct e1000_tx_desc *tx_desc; 3126 int dd_idx = adapter->tx_dd[adapter->tx_dd_head]; 3127 3128 tx_desc = &adapter->tx_desc_base[dd_idx]; 3129 if (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3130 EM_INC_TXDD_IDX(adapter->tx_dd_head); 3131 3132 if (++dd_idx == adapter->num_tx_desc) 3133 dd_idx = 0; 3134 3135 while (first != dd_idx) { 3136 logif(pkt_txclean); 3137 3138 KKASSERT(num_avail < adapter->num_tx_desc); 3139 num_avail++; 3140 3141 tx_buffer = &adapter->tx_buffer_area[first]; 3142 if (tx_buffer->m_head != NULL) 3143 em_free_txbuffer(adapter, tx_buffer); 3144 3145 if (++first == adapter->num_tx_desc) 3146 first = 0; 3147 } 3148 } else { 3149 break; 3150 } 3151 } 3152 adapter->next_tx_to_clean = first; 3153 adapter->num_tx_desc_avail = num_avail; 3154 3155 if (adapter->tx_dd_head == adapter->tx_dd_tail) { 3156 adapter->tx_dd_head = 0; 3157 adapter->tx_dd_tail = 0; 3158 } 3159 3160 if (!EM_IS_OACTIVE(adapter)) { 3161 ifq_clr_oactive(&ifp->if_snd); 3162 3163 /* All clean, turn off the timer */ 3164 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 3165 ifp->if_timer = 0; 3166 } 3167 adapter->tx_running = EM_TX_RUNNING; 3168 } 3169 3170 static void 3171 em_tx_collect(struct adapter *adapter, boolean_t gc) 3172 { 3173 struct ifnet *ifp = &adapter->arpcom.ac_if; 3174 struct em_buffer *tx_buffer; 3175 int tdh, first, num_avail, dd_idx = -1; 3176 3177 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 3178 return; 3179 3180 tdh = E1000_READ_REG(&adapter->hw, E1000_TDH(0)); 3181 if (tdh == adapter->next_tx_to_clean) { 3182 if (gc && adapter->tx_nmbuf > 0) 3183 adapter->tx_running = EM_TX_RUNNING; 3184 return; 3185 } 3186 if (gc) 3187 adapter->tx_gc++; 3188 3189 if (adapter->tx_dd_head != adapter->tx_dd_tail) 3190 dd_idx = adapter->tx_dd[adapter->tx_dd_head]; 3191 3192 num_avail = adapter->num_tx_desc_avail; 3193 first = adapter->next_tx_to_clean; 3194 3195 while (first != tdh) { 3196 logif(pkt_txclean); 3197 3198 KKASSERT(num_avail < adapter->num_tx_desc); 3199 num_avail++; 3200 3201 tx_buffer = &adapter->tx_buffer_area[first]; 3202 if (tx_buffer->m_head != NULL) 3203 em_free_txbuffer(adapter, tx_buffer); 3204 3205 if (first == dd_idx) { 3206 EM_INC_TXDD_IDX(adapter->tx_dd_head); 3207 if (adapter->tx_dd_head == adapter->tx_dd_tail) { 3208 adapter->tx_dd_head = 0; 3209 adapter->tx_dd_tail = 0; 3210 dd_idx = -1; 3211 } else { 3212 dd_idx = adapter->tx_dd[adapter->tx_dd_head]; 3213 } 3214 } 3215 3216 if (++first == adapter->num_tx_desc) 3217 first = 0; 3218 } 3219 adapter->next_tx_to_clean = first; 3220 adapter->num_tx_desc_avail = num_avail; 3221 3222 if (!EM_IS_OACTIVE(adapter)) { 3223 ifq_clr_oactive(&ifp->if_snd); 3224 3225 /* All clean, turn off the timer */ 3226 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 3227 ifp->if_timer = 0; 3228 } 3229 if (!gc || adapter->tx_nmbuf > 0) 3230 adapter->tx_running = EM_TX_RUNNING; 3231 } 3232 3233 /* 3234 * When Link is lost sometimes there is work still in the TX ring 3235 * which will result in a watchdog, rather than allow that do an 3236 * attempted cleanup and then reinit here. Note that this has been 3237 * seens mostly with fiber adapters. 3238 */ 3239 static void 3240 em_tx_purge(struct adapter *adapter) 3241 { 3242 struct ifnet *ifp = &adapter->arpcom.ac_if; 3243 3244 if (!adapter->link_active && ifp->if_timer) { 3245 em_tx_collect(adapter, FALSE); 3246 if (ifp->if_timer) { 3247 if_printf(ifp, "Link lost, TX pending, reinit\n"); 3248 ifp->if_timer = 0; 3249 em_init(adapter); 3250 } 3251 } 3252 } 3253 3254 static int 3255 em_newbuf(struct adapter *adapter, int i, int init) 3256 { 3257 struct mbuf *m; 3258 bus_dma_segment_t seg; 3259 bus_dmamap_t map; 3260 struct em_buffer *rx_buffer; 3261 int error, nseg; 3262 3263 m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR); 3264 if (m == NULL) { 3265 adapter->mbuf_cluster_failed++; 3266 if (init) { 3267 if_printf(&adapter->arpcom.ac_if, 3268 "Unable to allocate RX mbuf\n"); 3269 } 3270 return (ENOBUFS); 3271 } 3272 m->m_len = m->m_pkthdr.len = MCLBYTES; 3273 3274 if (adapter->hw.mac.max_frame_size <= MCLBYTES - ETHER_ALIGN) 3275 m_adj(m, ETHER_ALIGN); 3276 3277 error = bus_dmamap_load_mbuf_segment(adapter->rxtag, 3278 adapter->rx_sparemap, m, 3279 &seg, 1, &nseg, BUS_DMA_NOWAIT); 3280 if (error) { 3281 m_freem(m); 3282 if (init) { 3283 if_printf(&adapter->arpcom.ac_if, 3284 "Unable to load RX mbuf\n"); 3285 } 3286 return (error); 3287 } 3288 3289 rx_buffer = &adapter->rx_buffer_area[i]; 3290 if (rx_buffer->m_head != NULL) 3291 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 3292 3293 map = rx_buffer->map; 3294 rx_buffer->map = adapter->rx_sparemap; 3295 adapter->rx_sparemap = map; 3296 3297 rx_buffer->m_head = m; 3298 3299 adapter->rx_desc_base[i].buffer_addr = htole64(seg.ds_addr); 3300 return (0); 3301 } 3302 3303 static int 3304 em_create_rx_ring(struct adapter *adapter) 3305 { 3306 device_t dev = adapter->dev; 3307 struct em_buffer *rx_buffer; 3308 int i, error; 3309 3310 adapter->rx_buffer_area = 3311 kmalloc(sizeof(struct em_buffer) * adapter->num_rx_desc, 3312 M_DEVBUF, M_WAITOK | M_ZERO); 3313 3314 /* 3315 * Create DMA tag for rx buffers 3316 */ 3317 error = bus_dma_tag_create(adapter->parent_dtag, /* parent */ 3318 1, 0, /* alignment, bounds */ 3319 BUS_SPACE_MAXADDR, /* lowaddr */ 3320 BUS_SPACE_MAXADDR, /* highaddr */ 3321 NULL, NULL, /* filter, filterarg */ 3322 MCLBYTES, /* maxsize */ 3323 1, /* nsegments */ 3324 MCLBYTES, /* maxsegsize */ 3325 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */ 3326 &adapter->rxtag); 3327 if (error) { 3328 device_printf(dev, "Unable to allocate RX DMA tag\n"); 3329 kfree(adapter->rx_buffer_area, M_DEVBUF); 3330 adapter->rx_buffer_area = NULL; 3331 return error; 3332 } 3333 3334 /* 3335 * Create spare DMA map for rx buffers 3336 */ 3337 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK, 3338 &adapter->rx_sparemap); 3339 if (error) { 3340 device_printf(dev, "Unable to create spare RX DMA map\n"); 3341 bus_dma_tag_destroy(adapter->rxtag); 3342 kfree(adapter->rx_buffer_area, M_DEVBUF); 3343 adapter->rx_buffer_area = NULL; 3344 return error; 3345 } 3346 3347 /* 3348 * Create DMA maps for rx buffers 3349 */ 3350 for (i = 0; i < adapter->num_rx_desc; i++) { 3351 rx_buffer = &adapter->rx_buffer_area[i]; 3352 3353 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK, 3354 &rx_buffer->map); 3355 if (error) { 3356 device_printf(dev, "Unable to create RX DMA map\n"); 3357 em_destroy_rx_ring(adapter, i); 3358 return error; 3359 } 3360 } 3361 return (0); 3362 } 3363 3364 static int 3365 em_init_rx_ring(struct adapter *adapter) 3366 { 3367 int i, error; 3368 3369 /* Reset descriptor ring */ 3370 bzero(adapter->rx_desc_base, 3371 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); 3372 3373 /* Allocate new ones. */ 3374 for (i = 0; i < adapter->num_rx_desc; i++) { 3375 error = em_newbuf(adapter, i, 1); 3376 if (error) 3377 return (error); 3378 } 3379 3380 /* Setup our descriptor pointers */ 3381 adapter->next_rx_desc_to_check = 0; 3382 3383 return (0); 3384 } 3385 3386 static void 3387 em_init_rx_unit(struct adapter *adapter) 3388 { 3389 struct ifnet *ifp = &adapter->arpcom.ac_if; 3390 uint64_t bus_addr; 3391 uint32_t rctl; 3392 3393 /* 3394 * Make sure receives are disabled while setting 3395 * up the descriptor ring 3396 */ 3397 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 3398 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3399 3400 if (adapter->hw.mac.type >= e1000_82540) { 3401 uint32_t itr; 3402 3403 /* 3404 * Set the interrupt throttling rate. Value is calculated 3405 * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns) 3406 */ 3407 if (adapter->int_throttle_ceil) 3408 itr = 1000000000 / 256 / adapter->int_throttle_ceil; 3409 else 3410 itr = 0; 3411 em_set_itr(adapter, itr); 3412 } 3413 3414 /* Disable accelerated ackknowledge */ 3415 if (adapter->hw.mac.type == e1000_82574) { 3416 E1000_WRITE_REG(&adapter->hw, 3417 E1000_RFCTL, E1000_RFCTL_ACK_DIS); 3418 } 3419 3420 /* Receive Checksum Offload for TCP and UDP */ 3421 if (ifp->if_capenable & IFCAP_RXCSUM) { 3422 uint32_t rxcsum; 3423 3424 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 3425 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 3426 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 3427 } 3428 3429 /* 3430 * XXX TEMPORARY WORKAROUND: on some systems with 82573 3431 * long latencies are observed, like Lenovo X60. This 3432 * change eliminates the problem, but since having positive 3433 * values in RDTR is a known source of problems on other 3434 * platforms another solution is being sought. 3435 */ 3436 if (em_82573_workaround && adapter->hw.mac.type == e1000_82573) { 3437 E1000_WRITE_REG(&adapter->hw, E1000_RADV, EM_RADV_82573); 3438 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, EM_RDTR_82573); 3439 } 3440 3441 /* 3442 * Setup the Base and Length of the Rx Descriptor Ring 3443 */ 3444 bus_addr = adapter->rxdma.dma_paddr; 3445 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), 3446 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 3447 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), 3448 (uint32_t)(bus_addr >> 32)); 3449 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), 3450 (uint32_t)bus_addr); 3451 3452 /* 3453 * Setup the HW Rx Head and Tail Descriptor Pointers 3454 */ 3455 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); 3456 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1); 3457 3458 /* Set PTHRESH for improved jumbo performance */ 3459 if (((adapter->hw.mac.type == e1000_ich9lan) || 3460 (adapter->hw.mac.type == e1000_pch2lan) || 3461 (adapter->hw.mac.type == e1000_ich10lan)) && 3462 (ifp->if_mtu > ETHERMTU)) { 3463 uint32_t rxdctl; 3464 3465 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(0)); 3466 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(0), rxdctl | 3); 3467 } 3468 3469 if (adapter->hw.mac.type >= e1000_pch2lan) { 3470 if (ifp->if_mtu > ETHERMTU) 3471 e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, TRUE); 3472 else 3473 e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, FALSE); 3474 } 3475 3476 /* Setup the Receive Control Register */ 3477 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3478 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 3479 E1000_RCTL_RDMTS_HALF | 3480 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3481 3482 /* Make sure VLAN Filters are off */ 3483 rctl &= ~E1000_RCTL_VFE; 3484 3485 if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) 3486 rctl |= E1000_RCTL_SBP; 3487 else 3488 rctl &= ~E1000_RCTL_SBP; 3489 3490 switch (adapter->rx_buffer_len) { 3491 default: 3492 case 2048: 3493 rctl |= E1000_RCTL_SZ_2048; 3494 break; 3495 3496 case 4096: 3497 rctl |= E1000_RCTL_SZ_4096 | 3498 E1000_RCTL_BSEX | E1000_RCTL_LPE; 3499 break; 3500 3501 case 8192: 3502 rctl |= E1000_RCTL_SZ_8192 | 3503 E1000_RCTL_BSEX | E1000_RCTL_LPE; 3504 break; 3505 3506 case 16384: 3507 rctl |= E1000_RCTL_SZ_16384 | 3508 E1000_RCTL_BSEX | E1000_RCTL_LPE; 3509 break; 3510 } 3511 3512 if (ifp->if_mtu > ETHERMTU) 3513 rctl |= E1000_RCTL_LPE; 3514 else 3515 rctl &= ~E1000_RCTL_LPE; 3516 3517 /* Enable Receives */ 3518 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 3519 } 3520 3521 static void 3522 em_destroy_rx_ring(struct adapter *adapter, int ndesc) 3523 { 3524 struct em_buffer *rx_buffer; 3525 int i; 3526 3527 if (adapter->rx_buffer_area == NULL) 3528 return; 3529 3530 for (i = 0; i < ndesc; i++) { 3531 rx_buffer = &adapter->rx_buffer_area[i]; 3532 3533 KKASSERT(rx_buffer->m_head == NULL); 3534 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map); 3535 } 3536 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); 3537 bus_dma_tag_destroy(adapter->rxtag); 3538 3539 kfree(adapter->rx_buffer_area, M_DEVBUF); 3540 adapter->rx_buffer_area = NULL; 3541 } 3542 3543 static void 3544 em_rxeof(struct adapter *adapter, int count) 3545 { 3546 struct ifnet *ifp = &adapter->arpcom.ac_if; 3547 uint8_t status, accept_frame = 0, eop = 0; 3548 uint16_t len, desc_len, prev_len_adj; 3549 struct e1000_rx_desc *current_desc; 3550 struct mbuf *mp; 3551 int i; 3552 3553 i = adapter->next_rx_desc_to_check; 3554 current_desc = &adapter->rx_desc_base[i]; 3555 3556 if (!(current_desc->status & E1000_RXD_STAT_DD)) 3557 return; 3558 3559 while ((current_desc->status & E1000_RXD_STAT_DD) && count != 0) { 3560 struct mbuf *m = NULL; 3561 3562 logif(pkt_receive); 3563 3564 mp = adapter->rx_buffer_area[i].m_head; 3565 3566 /* 3567 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 3568 * needs to access the last received byte in the mbuf. 3569 */ 3570 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, 3571 BUS_DMASYNC_POSTREAD); 3572 3573 accept_frame = 1; 3574 prev_len_adj = 0; 3575 desc_len = le16toh(current_desc->length); 3576 status = current_desc->status; 3577 if (status & E1000_RXD_STAT_EOP) { 3578 count--; 3579 eop = 1; 3580 if (desc_len < ETHER_CRC_LEN) { 3581 len = 0; 3582 prev_len_adj = ETHER_CRC_LEN - desc_len; 3583 } else { 3584 len = desc_len - ETHER_CRC_LEN; 3585 } 3586 } else { 3587 eop = 0; 3588 len = desc_len; 3589 } 3590 3591 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 3592 uint8_t last_byte; 3593 uint32_t pkt_len = desc_len; 3594 3595 if (adapter->fmp != NULL) 3596 pkt_len += adapter->fmp->m_pkthdr.len; 3597 3598 last_byte = *(mtod(mp, caddr_t) + desc_len - 1); 3599 if (TBI_ACCEPT(&adapter->hw, status, 3600 current_desc->errors, pkt_len, last_byte, 3601 adapter->min_frame_size, 3602 adapter->hw.mac.max_frame_size)) { 3603 e1000_tbi_adjust_stats_82543(&adapter->hw, 3604 &adapter->stats, pkt_len, 3605 adapter->hw.mac.addr, 3606 adapter->hw.mac.max_frame_size); 3607 if (len > 0) 3608 len--; 3609 } else { 3610 accept_frame = 0; 3611 } 3612 } 3613 3614 if (accept_frame) { 3615 if (em_newbuf(adapter, i, 0) != 0) { 3616 IFNET_STAT_INC(ifp, iqdrops, 1); 3617 goto discard; 3618 } 3619 3620 /* Assign correct length to the current fragment */ 3621 mp->m_len = len; 3622 3623 if (adapter->fmp == NULL) { 3624 mp->m_pkthdr.len = len; 3625 adapter->fmp = mp; /* Store the first mbuf */ 3626 adapter->lmp = mp; 3627 } else { 3628 /* 3629 * Chain mbuf's together 3630 */ 3631 3632 /* 3633 * Adjust length of previous mbuf in chain if 3634 * we received less than 4 bytes in the last 3635 * descriptor. 3636 */ 3637 if (prev_len_adj > 0) { 3638 adapter->lmp->m_len -= prev_len_adj; 3639 adapter->fmp->m_pkthdr.len -= 3640 prev_len_adj; 3641 } 3642 adapter->lmp->m_next = mp; 3643 adapter->lmp = adapter->lmp->m_next; 3644 adapter->fmp->m_pkthdr.len += len; 3645 } 3646 3647 if (eop) { 3648 adapter->fmp->m_pkthdr.rcvif = ifp; 3649 IFNET_STAT_INC(ifp, ipackets, 1); 3650 3651 if (ifp->if_capenable & IFCAP_RXCSUM) { 3652 em_rxcsum(adapter, current_desc, 3653 adapter->fmp); 3654 } 3655 3656 if (status & E1000_RXD_STAT_VP) { 3657 adapter->fmp->m_pkthdr.ether_vlantag = 3658 (le16toh(current_desc->special) & 3659 E1000_RXD_SPC_VLAN_MASK); 3660 adapter->fmp->m_flags |= M_VLANTAG; 3661 } 3662 m = adapter->fmp; 3663 adapter->fmp = NULL; 3664 adapter->lmp = NULL; 3665 } 3666 } else { 3667 IFNET_STAT_INC(ifp, ierrors, 1); 3668 discard: 3669 #ifdef foo 3670 /* Reuse loaded DMA map and just update mbuf chain */ 3671 mp = adapter->rx_buffer_area[i].m_head; 3672 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 3673 mp->m_data = mp->m_ext.ext_buf; 3674 mp->m_next = NULL; 3675 if (adapter->hw.mac.max_frame_size <= 3676 (MCLBYTES - ETHER_ALIGN)) 3677 m_adj(mp, ETHER_ALIGN); 3678 #endif 3679 if (adapter->fmp != NULL) { 3680 m_freem(adapter->fmp); 3681 adapter->fmp = NULL; 3682 adapter->lmp = NULL; 3683 } 3684 m = NULL; 3685 } 3686 3687 /* Zero out the receive descriptors status. */ 3688 current_desc->status = 0; 3689 3690 if (m != NULL) 3691 ifp->if_input(ifp, m, NULL, -1); 3692 3693 /* Advance our pointers to the next descriptor. */ 3694 if (++i == adapter->num_rx_desc) 3695 i = 0; 3696 current_desc = &adapter->rx_desc_base[i]; 3697 } 3698 adapter->next_rx_desc_to_check = i; 3699 3700 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 3701 if (--i < 0) 3702 i = adapter->num_rx_desc - 1; 3703 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); 3704 } 3705 3706 static void 3707 em_rxcsum(struct adapter *adapter, struct e1000_rx_desc *rx_desc, 3708 struct mbuf *mp) 3709 { 3710 /* 82543 or newer only */ 3711 if (adapter->hw.mac.type < e1000_82543 || 3712 /* Ignore Checksum bit is set */ 3713 (rx_desc->status & E1000_RXD_STAT_IXSM)) 3714 return; 3715 3716 if ((rx_desc->status & E1000_RXD_STAT_IPCS) && 3717 !(rx_desc->errors & E1000_RXD_ERR_IPE)) { 3718 /* IP Checksum Good */ 3719 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; 3720 } 3721 3722 if ((rx_desc->status & E1000_RXD_STAT_TCPCS) && 3723 !(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 3724 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID | 3725 CSUM_PSEUDO_HDR | 3726 CSUM_FRAG_NOT_CHECKED; 3727 mp->m_pkthdr.csum_data = htons(0xffff); 3728 } 3729 } 3730 3731 static void 3732 em_enable_intr(struct adapter *adapter) 3733 { 3734 uint32_t ims_mask = IMS_ENABLE_MASK; 3735 3736 lwkt_serialize_handler_enable(adapter->arpcom.ac_if.if_serializer); 3737 3738 #if 0 3739 /* XXX MSIX */ 3740 if (adapter->hw.mac.type == e1000_82574) { 3741 E1000_WRITE_REG(&adapter->hw, EM_EIAC, EM_MSIX_MASK); 3742 ims_mask |= EM_MSIX_MASK; 3743 } 3744 #endif 3745 E1000_WRITE_REG(&adapter->hw, E1000_IMS, ims_mask); 3746 } 3747 3748 static void 3749 em_disable_intr(struct adapter *adapter) 3750 { 3751 uint32_t clear = 0xffffffff; 3752 3753 /* 3754 * The first version of 82542 had an errata where when link was forced 3755 * it would stay up even up even if the cable was disconnected. 3756 * Sequence errors were used to detect the disconnect and then the 3757 * driver would unforce the link. This code in the in the ISR. For 3758 * this to work correctly the Sequence error interrupt had to be 3759 * enabled all the time. 3760 */ 3761 if (adapter->hw.mac.type == e1000_82542 && 3762 adapter->hw.revision_id == E1000_REVISION_2) 3763 clear &= ~E1000_ICR_RXSEQ; 3764 else if (adapter->hw.mac.type == e1000_82574) 3765 E1000_WRITE_REG(&adapter->hw, EM_EIAC, 0); 3766 3767 E1000_WRITE_REG(&adapter->hw, E1000_IMC, clear); 3768 3769 adapter->npoll.ifpc_stcount = 0; 3770 3771 lwkt_serialize_handler_disable(adapter->arpcom.ac_if.if_serializer); 3772 } 3773 3774 /* 3775 * Bit of a misnomer, what this really means is 3776 * to enable OS management of the system... aka 3777 * to disable special hardware management features 3778 */ 3779 static void 3780 em_get_mgmt(struct adapter *adapter) 3781 { 3782 /* A shared code workaround */ 3783 #define E1000_82542_MANC2H E1000_MANC2H 3784 if (adapter->flags & EM_FLAG_HAS_MGMT) { 3785 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 3786 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 3787 3788 /* disable hardware interception of ARP */ 3789 manc &= ~(E1000_MANC_ARP_EN); 3790 3791 /* enable receiving management packets to the host */ 3792 if (adapter->hw.mac.type >= e1000_82571) { 3793 manc |= E1000_MANC_EN_MNG2HOST; 3794 #define E1000_MNG2HOST_PORT_623 (1 << 5) 3795 #define E1000_MNG2HOST_PORT_664 (1 << 6) 3796 manc2h |= E1000_MNG2HOST_PORT_623; 3797 manc2h |= E1000_MNG2HOST_PORT_664; 3798 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 3799 } 3800 3801 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 3802 } 3803 } 3804 3805 /* 3806 * Give control back to hardware management 3807 * controller if there is one. 3808 */ 3809 static void 3810 em_rel_mgmt(struct adapter *adapter) 3811 { 3812 if (adapter->flags & EM_FLAG_HAS_MGMT) { 3813 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 3814 3815 /* re-enable hardware interception of ARP */ 3816 manc |= E1000_MANC_ARP_EN; 3817 3818 if (adapter->hw.mac.type >= e1000_82571) 3819 manc &= ~E1000_MANC_EN_MNG2HOST; 3820 3821 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 3822 } 3823 } 3824 3825 /* 3826 * em_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3827 * For ASF and Pass Through versions of f/w this means that 3828 * the driver is loaded. For AMT version (only with 82573) 3829 * of the f/w this means that the network i/f is open. 3830 */ 3831 static void 3832 em_get_hw_control(struct adapter *adapter) 3833 { 3834 /* Let firmware know the driver has taken over */ 3835 if (adapter->hw.mac.type == e1000_82573) { 3836 uint32_t swsm; 3837 3838 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 3839 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 3840 swsm | E1000_SWSM_DRV_LOAD); 3841 } else { 3842 uint32_t ctrl_ext; 3843 3844 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 3845 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 3846 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 3847 } 3848 adapter->flags |= EM_FLAG_HW_CTRL; 3849 } 3850 3851 /* 3852 * em_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3853 * For ASF and Pass Through versions of f/w this means that the 3854 * driver is no longer loaded. For AMT version (only with 82573) 3855 * of the f/w this means that the network i/f is closed. 3856 */ 3857 static void 3858 em_rel_hw_control(struct adapter *adapter) 3859 { 3860 if ((adapter->flags & EM_FLAG_HW_CTRL) == 0) 3861 return; 3862 adapter->flags &= ~EM_FLAG_HW_CTRL; 3863 3864 /* Let firmware taken over control of h/w */ 3865 if (adapter->hw.mac.type == e1000_82573) { 3866 uint32_t swsm; 3867 3868 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 3869 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 3870 swsm & ~E1000_SWSM_DRV_LOAD); 3871 } else { 3872 uint32_t ctrl_ext; 3873 3874 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 3875 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 3876 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 3877 } 3878 } 3879 3880 static int 3881 em_is_valid_eaddr(const uint8_t *addr) 3882 { 3883 char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; 3884 3885 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3886 return (FALSE); 3887 3888 return (TRUE); 3889 } 3890 3891 /* 3892 * Enable PCI Wake On Lan capability 3893 */ 3894 static void 3895 em_enable_wol(device_t dev) 3896 { 3897 uint16_t cap, status; 3898 uint8_t id; 3899 3900 /* First find the capabilities pointer*/ 3901 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 3902 3903 /* Read the PM Capabilities */ 3904 id = pci_read_config(dev, cap, 1); 3905 if (id != PCIY_PMG) /* Something wrong */ 3906 return; 3907 3908 /* 3909 * OK, we have the power capabilities, 3910 * so now get the status register 3911 */ 3912 cap += PCIR_POWER_STATUS; 3913 status = pci_read_config(dev, cap, 2); 3914 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 3915 pci_write_config(dev, cap, status, 2); 3916 } 3917 3918 3919 /* 3920 * 82544 Coexistence issue workaround. 3921 * There are 2 issues. 3922 * 1. Transmit Hang issue. 3923 * To detect this issue, following equation can be used... 3924 * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 3925 * If SUM[3:0] is in between 1 to 4, we will have this issue. 3926 * 3927 * 2. DAC issue. 3928 * To detect this issue, following equation can be used... 3929 * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 3930 * If SUM[3:0] is in between 9 to c, we will have this issue. 3931 * 3932 * WORKAROUND: 3933 * Make sure we do not have ending address 3934 * as 1,2,3,4(Hang) or 9,a,b,c (DAC) 3935 */ 3936 static uint32_t 3937 em_82544_fill_desc(bus_addr_t address, uint32_t length, PDESC_ARRAY desc_array) 3938 { 3939 uint32_t safe_terminator; 3940 3941 /* 3942 * Since issue is sensitive to length and address. 3943 * Let us first check the address... 3944 */ 3945 if (length <= 4) { 3946 desc_array->descriptor[0].address = address; 3947 desc_array->descriptor[0].length = length; 3948 desc_array->elements = 1; 3949 return (desc_array->elements); 3950 } 3951 3952 safe_terminator = 3953 (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF); 3954 3955 /* If it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ 3956 if (safe_terminator == 0 || 3957 (safe_terminator > 4 && safe_terminator < 9) || 3958 (safe_terminator > 0xC && safe_terminator <= 0xF)) { 3959 desc_array->descriptor[0].address = address; 3960 desc_array->descriptor[0].length = length; 3961 desc_array->elements = 1; 3962 return (desc_array->elements); 3963 } 3964 3965 desc_array->descriptor[0].address = address; 3966 desc_array->descriptor[0].length = length - 4; 3967 desc_array->descriptor[1].address = address + (length - 4); 3968 desc_array->descriptor[1].length = 4; 3969 desc_array->elements = 2; 3970 return (desc_array->elements); 3971 } 3972 3973 static void 3974 em_update_stats(struct adapter *adapter) 3975 { 3976 struct ifnet *ifp = &adapter->arpcom.ac_if; 3977 3978 if (adapter->hw.phy.media_type == e1000_media_type_copper || 3979 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 3980 adapter->stats.symerrs += 3981 E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 3982 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 3983 } 3984 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 3985 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 3986 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 3987 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 3988 3989 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 3990 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 3991 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 3992 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 3993 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 3994 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 3995 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 3996 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 3997 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 3998 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 3999 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4000 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4001 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4002 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4003 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4004 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4005 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4006 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4007 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4008 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4009 4010 /* For the 64-bit byte counters the low dword must be read first. */ 4011 /* Both registers clear on the read of the high dword */ 4012 4013 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4014 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4015 4016 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4017 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4018 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4019 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4020 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4021 4022 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4023 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4024 4025 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4026 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4027 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4028 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4029 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4030 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4031 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4032 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4033 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4034 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4035 4036 if (adapter->hw.mac.type >= e1000_82543) { 4037 adapter->stats.algnerrc += 4038 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4039 adapter->stats.rxerrc += 4040 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4041 adapter->stats.tncrs += 4042 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4043 adapter->stats.cexterr += 4044 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4045 adapter->stats.tsctc += 4046 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4047 adapter->stats.tsctfc += 4048 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4049 } 4050 4051 IFNET_STAT_SET(ifp, collisions, adapter->stats.colc); 4052 4053 /* Rx Errors */ 4054 IFNET_STAT_SET(ifp, ierrors, 4055 adapter->dropped_pkts + adapter->stats.rxerrc + 4056 adapter->stats.crcerrs + adapter->stats.algnerrc + 4057 adapter->stats.ruc + adapter->stats.roc + 4058 adapter->stats.mpc + adapter->stats.cexterr); 4059 4060 /* Tx Errors */ 4061 IFNET_STAT_SET(ifp, oerrors, 4062 adapter->stats.ecol + adapter->stats.latecol + 4063 adapter->watchdog_events); 4064 } 4065 4066 static void 4067 em_print_debug_info(struct adapter *adapter) 4068 { 4069 device_t dev = adapter->dev; 4070 uint8_t *hw_addr = adapter->hw.hw_addr; 4071 4072 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4073 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4074 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4075 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4076 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4077 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4078 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4079 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4080 adapter->hw.fc.high_water, 4081 adapter->hw.fc.low_water); 4082 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 4083 E1000_READ_REG(&adapter->hw, E1000_TIDV), 4084 E1000_READ_REG(&adapter->hw, E1000_TADV)); 4085 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 4086 E1000_READ_REG(&adapter->hw, E1000_RDTR), 4087 E1000_READ_REG(&adapter->hw, E1000_RADV)); 4088 device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n", 4089 (long long)adapter->tx_fifo_wrk_cnt, 4090 (long long)adapter->tx_fifo_reset_cnt); 4091 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 4092 E1000_READ_REG(&adapter->hw, E1000_TDH(0)), 4093 E1000_READ_REG(&adapter->hw, E1000_TDT(0))); 4094 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 4095 E1000_READ_REG(&adapter->hw, E1000_RDH(0)), 4096 E1000_READ_REG(&adapter->hw, E1000_RDT(0))); 4097 device_printf(dev, "Num Tx descriptors avail = %d\n", 4098 adapter->num_tx_desc_avail); 4099 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 4100 adapter->no_tx_desc_avail1); 4101 device_printf(dev, "Tx Descriptors not avail2 = %ld\n", 4102 adapter->no_tx_desc_avail2); 4103 device_printf(dev, "Std mbuf failed = %ld\n", 4104 adapter->mbuf_alloc_failed); 4105 device_printf(dev, "Std mbuf cluster failed = %ld\n", 4106 adapter->mbuf_cluster_failed); 4107 device_printf(dev, "Driver dropped packets = %ld\n", 4108 adapter->dropped_pkts); 4109 device_printf(dev, "Driver tx dma failure in encap = %ld\n", 4110 adapter->no_tx_dma_setup); 4111 } 4112 4113 static void 4114 em_print_hw_stats(struct adapter *adapter) 4115 { 4116 device_t dev = adapter->dev; 4117 4118 device_printf(dev, "Excessive collisions = %lld\n", 4119 (long long)adapter->stats.ecol); 4120 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 4121 device_printf(dev, "Symbol errors = %lld\n", 4122 (long long)adapter->stats.symerrs); 4123 #endif 4124 device_printf(dev, "Sequence errors = %lld\n", 4125 (long long)adapter->stats.sec); 4126 device_printf(dev, "Defer count = %lld\n", 4127 (long long)adapter->stats.dc); 4128 device_printf(dev, "Missed Packets = %lld\n", 4129 (long long)adapter->stats.mpc); 4130 device_printf(dev, "Receive No Buffers = %lld\n", 4131 (long long)adapter->stats.rnbc); 4132 /* RLEC is inaccurate on some hardware, calculate our own. */ 4133 device_printf(dev, "Receive Length Errors = %lld\n", 4134 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4135 device_printf(dev, "Receive errors = %lld\n", 4136 (long long)adapter->stats.rxerrc); 4137 device_printf(dev, "Crc errors = %lld\n", 4138 (long long)adapter->stats.crcerrs); 4139 device_printf(dev, "Alignment errors = %lld\n", 4140 (long long)adapter->stats.algnerrc); 4141 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4142 (long long)adapter->stats.cexterr); 4143 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 4144 device_printf(dev, "watchdog timeouts = %ld\n", 4145 adapter->watchdog_events); 4146 device_printf(dev, "XON Rcvd = %lld\n", 4147 (long long)adapter->stats.xonrxc); 4148 device_printf(dev, "XON Xmtd = %lld\n", 4149 (long long)adapter->stats.xontxc); 4150 device_printf(dev, "XOFF Rcvd = %lld\n", 4151 (long long)adapter->stats.xoffrxc); 4152 device_printf(dev, "XOFF Xmtd = %lld\n", 4153 (long long)adapter->stats.xofftxc); 4154 device_printf(dev, "Good Packets Rcvd = %lld\n", 4155 (long long)adapter->stats.gprc); 4156 device_printf(dev, "Good Packets Xmtd = %lld\n", 4157 (long long)adapter->stats.gptc); 4158 } 4159 4160 static void 4161 em_print_nvm_info(struct adapter *adapter) 4162 { 4163 uint16_t eeprom_data; 4164 int i, j, row = 0; 4165 4166 /* Its a bit crude, but it gets the job done */ 4167 kprintf("\nInterface EEPROM Dump:\n"); 4168 kprintf("Offset\n0x0000 "); 4169 for (i = 0, j = 0; i < 32; i++, j++) { 4170 if (j == 8) { /* Make the offset block */ 4171 j = 0; ++row; 4172 kprintf("\n0x00%x0 ",row); 4173 } 4174 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4175 kprintf("%04x ", eeprom_data); 4176 } 4177 kprintf("\n"); 4178 } 4179 4180 static int 4181 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4182 { 4183 struct adapter *adapter; 4184 struct ifnet *ifp; 4185 int error, result; 4186 4187 result = -1; 4188 error = sysctl_handle_int(oidp, &result, 0, req); 4189 if (error || !req->newptr) 4190 return (error); 4191 4192 adapter = (struct adapter *)arg1; 4193 ifp = &adapter->arpcom.ac_if; 4194 4195 lwkt_serialize_enter(ifp->if_serializer); 4196 4197 if (result == 1) 4198 em_print_debug_info(adapter); 4199 4200 /* 4201 * This value will cause a hex dump of the 4202 * first 32 16-bit words of the EEPROM to 4203 * the screen. 4204 */ 4205 if (result == 2) 4206 em_print_nvm_info(adapter); 4207 4208 lwkt_serialize_exit(ifp->if_serializer); 4209 4210 return (error); 4211 } 4212 4213 static int 4214 em_sysctl_stats(SYSCTL_HANDLER_ARGS) 4215 { 4216 int error, result; 4217 4218 result = -1; 4219 error = sysctl_handle_int(oidp, &result, 0, req); 4220 if (error || !req->newptr) 4221 return (error); 4222 4223 if (result == 1) { 4224 struct adapter *adapter = (struct adapter *)arg1; 4225 struct ifnet *ifp = &adapter->arpcom.ac_if; 4226 4227 lwkt_serialize_enter(ifp->if_serializer); 4228 em_print_hw_stats(adapter); 4229 lwkt_serialize_exit(ifp->if_serializer); 4230 } 4231 return (error); 4232 } 4233 4234 static void 4235 em_add_sysctl(struct adapter *adapter) 4236 { 4237 struct sysctl_ctx_list *ctx; 4238 struct sysctl_oid *tree; 4239 4240 ctx = device_get_sysctl_ctx(adapter->dev); 4241 tree = device_get_sysctl_tree(adapter->dev); 4242 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4243 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 4244 em_sysctl_debug_info, "I", "Debug Information"); 4245 4246 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4247 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 4248 em_sysctl_stats, "I", "Statistics"); 4249 4250 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4251 OID_AUTO, "rxd", CTLFLAG_RD, 4252 &adapter->num_rx_desc, 0, NULL); 4253 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4254 OID_AUTO, "txd", CTLFLAG_RD, 4255 &adapter->num_tx_desc, 0, NULL); 4256 4257 if (adapter->hw.mac.type >= e1000_82540) { 4258 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4259 OID_AUTO, "int_throttle_ceil", 4260 CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 4261 em_sysctl_int_throttle, "I", 4262 "interrupt throttling rate"); 4263 } 4264 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), 4265 OID_AUTO, "int_tx_nsegs", 4266 CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 4267 em_sysctl_int_tx_nsegs, "I", 4268 "# segments per TX interrupt"); 4269 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), 4270 OID_AUTO, "wreg_tx_nsegs", CTLFLAG_RW, 4271 &adapter->tx_wreg_nsegs, 0, 4272 "# segments before write to hardware register"); 4273 SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "tx_nmbuf", 4274 CTLFLAG_RD, &adapter->tx_nmbuf, 0, "# of pending TX mbufs"); 4275 SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "tx_gc", 4276 CTLFLAG_RW, &adapter->tx_gc, "# of TX GC"); 4277 } 4278 4279 static int 4280 em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS) 4281 { 4282 struct adapter *adapter = (void *)arg1; 4283 struct ifnet *ifp = &adapter->arpcom.ac_if; 4284 int error, throttle; 4285 4286 throttle = adapter->int_throttle_ceil; 4287 error = sysctl_handle_int(oidp, &throttle, 0, req); 4288 if (error || req->newptr == NULL) 4289 return error; 4290 if (throttle < 0 || throttle > 1000000000 / 256) 4291 return EINVAL; 4292 4293 if (throttle) { 4294 /* 4295 * Set the interrupt throttling rate in 256ns increments, 4296 * recalculate sysctl value assignment to get exact frequency. 4297 */ 4298 throttle = 1000000000 / 256 / throttle; 4299 4300 /* Upper 16bits of ITR is reserved and should be zero */ 4301 if (throttle & 0xffff0000) 4302 return EINVAL; 4303 } 4304 4305 lwkt_serialize_enter(ifp->if_serializer); 4306 4307 if (throttle) 4308 adapter->int_throttle_ceil = 1000000000 / 256 / throttle; 4309 else 4310 adapter->int_throttle_ceil = 0; 4311 4312 if (ifp->if_flags & IFF_RUNNING) 4313 em_set_itr(adapter, throttle); 4314 4315 lwkt_serialize_exit(ifp->if_serializer); 4316 4317 if (bootverbose) { 4318 if_printf(ifp, "Interrupt moderation set to %d/sec\n", 4319 adapter->int_throttle_ceil); 4320 } 4321 return 0; 4322 } 4323 4324 static int 4325 em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS) 4326 { 4327 struct adapter *adapter = (void *)arg1; 4328 struct ifnet *ifp = &adapter->arpcom.ac_if; 4329 int error, segs; 4330 4331 segs = adapter->tx_int_nsegs; 4332 error = sysctl_handle_int(oidp, &segs, 0, req); 4333 if (error || req->newptr == NULL) 4334 return error; 4335 if (segs <= 0) 4336 return EINVAL; 4337 4338 lwkt_serialize_enter(ifp->if_serializer); 4339 4340 /* 4341 * Don't allow int_tx_nsegs to become: 4342 * o Less the oact_tx_desc 4343 * o Too large that no TX desc will cause TX interrupt to 4344 * be generated (OACTIVE will never recover) 4345 * o Too small that will cause tx_dd[] overflow 4346 */ 4347 if (segs < adapter->oact_tx_desc || 4348 segs >= adapter->num_tx_desc - adapter->oact_tx_desc || 4349 segs < adapter->num_tx_desc / EM_TXDD_SAFE) { 4350 error = EINVAL; 4351 } else { 4352 error = 0; 4353 adapter->tx_int_nsegs = segs; 4354 } 4355 4356 lwkt_serialize_exit(ifp->if_serializer); 4357 4358 return error; 4359 } 4360 4361 static void 4362 em_set_itr(struct adapter *adapter, uint32_t itr) 4363 { 4364 E1000_WRITE_REG(&adapter->hw, E1000_ITR, itr); 4365 if (adapter->hw.mac.type == e1000_82574) { 4366 int i; 4367 4368 /* 4369 * When using MSIX interrupts we need to 4370 * throttle using the EITR register 4371 */ 4372 for (i = 0; i < 4; ++i) { 4373 E1000_WRITE_REG(&adapter->hw, 4374 E1000_EITR_82574(i), itr); 4375 } 4376 } 4377 } 4378 4379 static void 4380 em_disable_aspm(struct adapter *adapter) 4381 { 4382 uint16_t link_cap, link_ctrl, disable; 4383 uint8_t pcie_ptr, reg; 4384 device_t dev = adapter->dev; 4385 4386 switch (adapter->hw.mac.type) { 4387 case e1000_82571: 4388 case e1000_82572: 4389 case e1000_82573: 4390 /* 4391 * 82573 specification update 4392 * errata #8 disable L0s 4393 * errata #41 disable L1 4394 * 4395 * 82571/82572 specification update 4396 # errata #13 disable L1 4397 * errata #68 disable L0s 4398 */ 4399 disable = PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1; 4400 break; 4401 4402 case e1000_82574: 4403 case e1000_82583: 4404 /* 4405 * 82574 specification update errata #20 4406 * 82583 specification update errata #9 4407 * 4408 * There is no need to disable L1 4409 */ 4410 disable = PCIEM_LNKCTL_ASPM_L0S; 4411 break; 4412 4413 default: 4414 return; 4415 } 4416 4417 pcie_ptr = pci_get_pciecap_ptr(dev); 4418 if (pcie_ptr == 0) 4419 return; 4420 4421 link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2); 4422 if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0) 4423 return; 4424 4425 if (bootverbose) { 4426 if_printf(&adapter->arpcom.ac_if, 4427 "disable ASPM %#02x\n", disable); 4428 } 4429 4430 reg = pcie_ptr + PCIER_LINKCTRL; 4431 link_ctrl = pci_read_config(dev, reg, 2); 4432 link_ctrl &= ~disable; 4433 pci_write_config(dev, reg, link_ctrl, 2); 4434 } 4435 4436 static int 4437 em_tso_pullup(struct adapter *adapter, struct mbuf **mp) 4438 { 4439 int iphlen, hoff, thoff, ex = 0; 4440 struct mbuf *m; 4441 struct ip *ip; 4442 4443 m = *mp; 4444 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable")); 4445 4446 iphlen = m->m_pkthdr.csum_iphlen; 4447 thoff = m->m_pkthdr.csum_thlen; 4448 hoff = m->m_pkthdr.csum_lhlen; 4449 4450 KASSERT(iphlen > 0, ("invalid ip hlen")); 4451 KASSERT(thoff > 0, ("invalid tcp hlen")); 4452 KASSERT(hoff > 0, ("invalid ether hlen")); 4453 4454 if (adapter->flags & EM_FLAG_TSO_PULLEX) 4455 ex = 4; 4456 4457 if (m->m_len < hoff + iphlen + thoff + ex) { 4458 m = m_pullup(m, hoff + iphlen + thoff + ex); 4459 if (m == NULL) { 4460 *mp = NULL; 4461 return ENOBUFS; 4462 } 4463 *mp = m; 4464 } 4465 ip = mtodoff(m, struct ip *, hoff); 4466 ip->ip_len = 0; 4467 4468 return 0; 4469 } 4470 4471 static int 4472 em_tso_setup(struct adapter *adapter, struct mbuf *mp, 4473 uint32_t *txd_upper, uint32_t *txd_lower) 4474 { 4475 struct e1000_context_desc *TXD; 4476 int hoff, iphlen, thoff, hlen; 4477 int mss, pktlen, curr_txd; 4478 4479 iphlen = mp->m_pkthdr.csum_iphlen; 4480 thoff = mp->m_pkthdr.csum_thlen; 4481 hoff = mp->m_pkthdr.csum_lhlen; 4482 mss = mp->m_pkthdr.tso_segsz; 4483 pktlen = mp->m_pkthdr.len; 4484 4485 if (adapter->csum_flags == CSUM_TSO && 4486 adapter->csum_iphlen == iphlen && 4487 adapter->csum_lhlen == hoff && 4488 adapter->csum_thlen == thoff && 4489 adapter->csum_mss == mss && 4490 adapter->csum_pktlen == pktlen) { 4491 *txd_upper = adapter->csum_txd_upper; 4492 *txd_lower = adapter->csum_txd_lower; 4493 return 0; 4494 } 4495 hlen = hoff + iphlen + thoff; 4496 4497 /* 4498 * Setup a new TSO context. 4499 */ 4500 4501 curr_txd = adapter->next_avail_tx_desc; 4502 TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd]; 4503 4504 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 4505 E1000_TXD_DTYP_D | /* Data descr type */ 4506 E1000_TXD_CMD_TSE; /* Do TSE on this packet */ 4507 4508 /* IP and/or TCP header checksum calculation and insertion. */ 4509 *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 4510 4511 /* 4512 * Start offset for header checksum calculation. 4513 * End offset for header checksum calculation. 4514 * Offset of place put the checksum. 4515 */ 4516 TXD->lower_setup.ip_fields.ipcss = hoff; 4517 TXD->lower_setup.ip_fields.ipcse = htole16(hoff + iphlen - 1); 4518 TXD->lower_setup.ip_fields.ipcso = hoff + offsetof(struct ip, ip_sum); 4519 4520 /* 4521 * Start offset for payload checksum calculation. 4522 * End offset for payload checksum calculation. 4523 * Offset of place to put the checksum. 4524 */ 4525 TXD->upper_setup.tcp_fields.tucss = hoff + iphlen; 4526 TXD->upper_setup.tcp_fields.tucse = 0; 4527 TXD->upper_setup.tcp_fields.tucso = 4528 hoff + iphlen + offsetof(struct tcphdr, th_sum); 4529 4530 /* 4531 * Payload size per packet w/o any headers. 4532 * Length of all headers up to payload. 4533 */ 4534 TXD->tcp_seg_setup.fields.mss = htole16(mss); 4535 TXD->tcp_seg_setup.fields.hdr_len = hlen; 4536 TXD->cmd_and_length = htole32(E1000_TXD_CMD_IFCS | 4537 E1000_TXD_CMD_DEXT | /* Extended descr */ 4538 E1000_TXD_CMD_TSE | /* TSE context */ 4539 E1000_TXD_CMD_IP | /* Do IP csum */ 4540 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 4541 (pktlen - hlen)); /* Total len */ 4542 4543 /* Save the information for this TSO context */ 4544 adapter->csum_flags = CSUM_TSO; 4545 adapter->csum_lhlen = hoff; 4546 adapter->csum_iphlen = iphlen; 4547 adapter->csum_thlen = thoff; 4548 adapter->csum_mss = mss; 4549 adapter->csum_pktlen = pktlen; 4550 adapter->csum_txd_upper = *txd_upper; 4551 adapter->csum_txd_lower = *txd_lower; 4552 4553 if (++curr_txd == adapter->num_tx_desc) 4554 curr_txd = 0; 4555 4556 KKASSERT(adapter->num_tx_desc_avail > 0); 4557 adapter->num_tx_desc_avail--; 4558 4559 adapter->next_avail_tx_desc = curr_txd; 4560 return 1; 4561 } 4562