1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 #if 0 33 __FBSDID("$FreeBSD: head/sys/dev/ena/ena.c 333456 2018-05-10 09:37:54Z mw $"); 34 #endif 35 __KERNEL_RCSID(0, "$NetBSD: if_ena.c,v 1.41 2023/12/20 18:09:19 skrll Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/bus.h> 40 #include <sys/endian.h> 41 #include <sys/kernel.h> 42 #include <sys/kmem.h> 43 #include <sys/kthread.h> 44 #include <sys/mbuf.h> 45 #include <sys/module.h> 46 #include <sys/socket.h> 47 #include <sys/sockio.h> 48 #include <sys/sysctl.h> 49 #include <sys/time.h> 50 #include <sys/workqueue.h> 51 #include <sys/callout.h> 52 #include <sys/interrupt.h> 53 #include <sys/cpu.h> 54 55 #include <net/if_ether.h> 56 #include <net/if_vlanvar.h> 57 58 #include <dev/pci/if_enavar.h> 59 60 /********************************************************* 61 * Function prototypes 62 *********************************************************/ 63 /* cfattach interface functions */ 64 static int ena_probe(device_t, cfdata_t, void *); 65 static void ena_attach(device_t, device_t, void *); 66 static int ena_detach(device_t, int); 67 68 /* ifnet interface functions */ 69 static int ena_init(struct ifnet *); 70 static void ena_stop(struct ifnet *, int); 71 static int ena_ioctl(struct ifnet *, u_long, void *); 72 static int ena_media_change(struct ifnet *); 73 static void ena_media_status(struct ifnet *, struct ifmediareq *); 74 static int ena_mq_start(struct ifnet *, struct mbuf *); 75 76 /* attach or detach */ 77 static int ena_calc_io_queue_num(struct pci_attach_args *, 78 struct ena_adapter *, 79 struct ena_com_dev_get_features_ctx *); 80 static int ena_calc_queue_size(struct ena_adapter *, uint16_t *, 81 uint16_t *, struct ena_com_dev_get_features_ctx *); 82 83 static int ena_allocate_pci_resources(struct pci_attach_args *, 84 struct ena_adapter *); 85 static void ena_free_pci_resources(struct ena_adapter *); 86 static void ena_free_irqs(struct ena_adapter*); 87 88 static void ena_init_io_rings_common(struct ena_adapter *, 89 struct ena_ring *, uint16_t); 90 static void ena_init_io_rings(struct ena_adapter *); 91 static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int); 92 static void ena_free_all_io_rings_resources(struct ena_adapter *); 93 94 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); 95 static int ena_setup_ifnet(device_t, struct ena_adapter *, 96 struct ena_com_dev_get_features_ctx *); 97 static void ena_rss_init_default(device_t); 98 99 static inline void ena_alloc_counters_rx(struct ena_adapter *, 100 struct ena_stats_rx *, int); 101 static inline void ena_alloc_counters_tx(struct ena_adapter *, 102 struct ena_stats_tx *, int); 103 static inline void ena_alloc_counters_dev(struct ena_adapter *, 104 struct ena_stats_dev *, int); 105 static inline void ena_alloc_counters_hwstats(struct ena_adapter *, 106 struct ena_hw_stats *, int); 107 static inline void ena_free_counters(struct evcnt *, int, int); 108 109 /* attach or detach or ena_reset_task() */ 110 static void ena_reset_task(struct work *, void *); 111 112 static void ena_free_mgmnt_irq(struct ena_adapter *); 113 static void ena_disable_msix(struct ena_adapter *); 114 static void ena_config_host_info(struct ena_com_dev *); 115 static int ena_device_init(struct ena_adapter *, device_t, 116 struct ena_com_dev_get_features_ctx *, int *); 117 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *, 118 int); 119 static int ena_enable_msix(struct ena_adapter *); 120 static int ena_request_mgmnt_irq(struct ena_adapter *); 121 122 /* I/F up or down */ 123 static int ena_up_complete(struct ena_adapter *); 124 static int ena_up(struct ena_adapter *); 125 static void ena_down(struct ena_adapter *); 126 static void ena_set_stopping_flag(struct ena_adapter *, bool); 127 128 static int ena_setup_rx_resources(struct ena_adapter *, unsigned int); 129 static int ena_setup_all_rx_resources(struct ena_adapter *); 130 static void ena_free_rx_resources(struct ena_adapter *, unsigned int); 131 static void ena_free_all_rx_resources(struct ena_adapter *); 132 static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *, 133 struct ena_rx_buffer *); 134 static void ena_free_rx_bufs(struct ena_adapter *, unsigned int); 135 static void ena_free_all_rx_bufs(struct ena_adapter *); 136 137 static int ena_setup_tx_resources(struct ena_adapter *, int); 138 static int ena_setup_all_tx_resources(struct ena_adapter *); 139 static void ena_free_tx_resources(struct ena_adapter *, int); 140 static void ena_free_all_tx_resources(struct ena_adapter *); 141 static void ena_free_tx_bufs(struct ena_adapter *, unsigned int); 142 static void ena_free_all_tx_bufs(struct ena_adapter *); 143 144 static int ena_request_io_irq(struct ena_adapter *); 145 static void ena_free_io_irq(struct ena_adapter *); 146 static int ena_create_io_queues(struct ena_adapter *); 147 static void ena_destroy_all_tx_queues(struct ena_adapter *); 148 static void ena_destroy_all_rx_queues(struct ena_adapter *); 149 static void ena_destroy_all_io_queues(struct ena_adapter *); 150 151 static void ena_update_hwassist(struct ena_adapter *); 152 static int ena_rss_configure(struct ena_adapter *); 153 static void ena_unmask_all_io_irqs(struct ena_adapter *); 154 static inline void ena_reset_counters(struct evcnt *, int, int); 155 156 /* other hardware interrupt, workqueue, softint context */ 157 static int ena_intr_msix_mgmnt(void *); 158 static void ena_update_on_link_change(void *, 159 struct ena_admin_aenq_entry *); 160 static void ena_keep_alive_wd(void *, 161 struct ena_admin_aenq_entry *); 162 static void unimplemented_aenq_handler(void *, 163 struct ena_admin_aenq_entry *); 164 165 static int ena_handle_msix(void *); 166 static void ena_cleanup(struct work *, void *); 167 static inline int validate_rx_req_id(struct ena_ring *, uint16_t); 168 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, 169 struct ena_ring *, struct ena_rx_buffer *); 170 static int ena_refill_rx_bufs(struct ena_ring *, uint32_t); 171 static void ena_refill_all_rx_bufs(struct ena_adapter *); 172 static int ena_rx_cleanup(struct ena_ring *); 173 static struct mbuf* ena_rx_mbuf(struct ena_ring *, 174 struct ena_com_rx_buf_info *, 175 struct ena_com_rx_ctx *, uint16_t *); 176 static inline void ena_rx_checksum(struct ena_ring *, 177 struct ena_com_rx_ctx *, struct mbuf *); 178 179 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, 180 struct mbuf **mbuf); 181 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **); 182 static void ena_start_xmit(struct ena_ring *); 183 static void ena_deferred_mq_start(struct work *, void *); 184 static int ena_tx_cleanup(struct ena_ring *); 185 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *); 186 static inline int validate_tx_req_id(struct ena_ring *, uint16_t); 187 188 /* other */ 189 static int ena_change_mtu(struct ifnet *, int); 190 191 static void ena_timer_service(void *); 192 static void check_for_missing_keep_alive(struct ena_adapter *); 193 static void check_for_admin_com_state(struct ena_adapter *); 194 static int check_missing_comp_in_queue(struct ena_adapter *, struct ena_ring*); 195 static void check_for_missing_tx_completions(struct ena_adapter *); 196 static void check_for_empty_rx_ring(struct ena_adapter *); 197 static void ena_update_host_info(struct ena_admin_host_info *, 198 struct ifnet *); 199 200 #if 0 201 static int ena_setup_tx_dma_tag(struct ena_adapter *); 202 static int ena_free_tx_dma_tag(struct ena_adapter *); 203 static int ena_setup_rx_dma_tag(struct ena_adapter *); 204 static int ena_free_rx_dma_tag(struct ena_adapter *); 205 static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *, 206 struct mbuf *); 207 static uint64_t ena_get_counter(struct ifnet *, ift_counter); 208 static void ena_qflush(struct ifnet *); 209 #endif 210 211 static const char ena_version[] = 212 DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION; 213 214 #if 0 215 static SYSCTL_NODE(_hw, OID_AUTO, ena, CTLFLAG_RD, 0, "ENA driver parameters"); 216 #endif 217 218 /* 219 * Tuneable number of buffers in the buf-ring (drbr) 220 */ 221 static int ena_buf_ring_size = 4096; 222 #if 0 223 SYSCTL_INT(_hw_ena, OID_AUTO, buf_ring_size, CTLFLAG_RWTUN, 224 &ena_buf_ring_size, 0, "Size of the bufring"); 225 #endif 226 227 /* 228 * Logging level for changing verbosity of the output 229 */ 230 int ena_log_level = ENA_ALERT | ENA_WARNING; 231 #if 0 232 SYSCTL_INT(_hw_ena, OID_AUTO, log_level, CTLFLAG_RWTUN, 233 &ena_log_level, 0, "Logging level indicating verbosity of the logs"); 234 #endif 235 236 static const ena_vendor_info_t ena_vendor_info_array[] = { 237 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0}, 238 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_PF, 0}, 239 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0}, 240 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_VF, 0}, 241 /* Last entry */ 242 { 0, 0, 0 } 243 }; 244 245 /* 246 * Contains pointers to event handlers, e.g. link state change. 247 */ 248 static struct ena_aenq_handlers aenq_handlers; 249 250 int 251 ena_dma_alloc(device_t dmadev, bus_size_t size, 252 ena_mem_handle_t *dma , int mapflags) 253 { 254 struct ena_adapter *adapter = device_private(dmadev); 255 bus_size_t maxsize; 256 int error; 257 258 maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; 259 260 #if 0 261 /* XXX what is this needed for ? */ 262 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); 263 if (unlikely(dma_space_addr == 0)) 264 dma_space_addr = BUS_SPACE_MAXADDR; 265 #endif 266 267 dma->tag = adapter->sc_dmat; 268 269 if ((error = bus_dmamap_create(dma->tag, maxsize, 1, maxsize, 0, 270 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->map)) != 0) { 271 ena_trace(ENA_ALERT, "bus_dmamap_create(%ju) failed: %d\n", 272 (uintmax_t)maxsize, error); 273 goto fail_create; 274 } 275 276 error = bus_dmamem_alloc(dma->tag, maxsize, 8, 0, &dma->seg, 1, &dma->nseg, 277 BUS_DMA_ALLOCNOW); 278 if (error) { 279 ena_trace(ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n", 280 (uintmax_t)maxsize, error); 281 goto fail_alloc; 282 } 283 284 error = bus_dmamem_map(dma->tag, &dma->seg, dma->nseg, maxsize, 285 &dma->vaddr, BUS_DMA_COHERENT); 286 if (error) { 287 ena_trace(ENA_ALERT, "bus_dmamem_map(%ju) failed: %d\n", 288 (uintmax_t)maxsize, error); 289 goto fail_map; 290 } 291 memset(dma->vaddr, 0, maxsize); 292 293 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, 294 maxsize, NULL, mapflags); 295 if (error) { 296 ena_trace(ENA_ALERT, ": bus_dmamap_load failed: %d\n", error); 297 goto fail_load; 298 } 299 dma->paddr = dma->map->dm_segs[0].ds_addr; 300 301 return (0); 302 303 fail_load: 304 bus_dmamem_unmap(dma->tag, dma->vaddr, maxsize); 305 fail_map: 306 bus_dmamem_free(dma->tag, &dma->seg, dma->nseg); 307 fail_alloc: 308 bus_dmamap_destroy(adapter->sc_dmat, dma->map); 309 fail_create: 310 return (error); 311 } 312 313 static int 314 ena_allocate_pci_resources(struct pci_attach_args *pa, 315 struct ena_adapter *adapter) 316 { 317 pcireg_t memtype, reg; 318 int flags, error; 319 int msixoff; 320 321 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, ENA_REG_BAR); 322 if (PCI_MAPREG_TYPE(memtype) != PCI_MAPREG_TYPE_MEM) { 323 aprint_error_dev(adapter->pdev, "invalid type (type=0x%x)\n", 324 memtype); 325 return ENXIO; 326 } 327 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 328 if (((reg & PCI_COMMAND_MASTER_ENABLE) == 0) || 329 ((reg & PCI_COMMAND_MEM_ENABLE) == 0)) { 330 /* 331 * Enable address decoding for memory range in case BIOS or 332 * UEFI didn't set it. 333 */ 334 reg |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 335 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 336 reg); 337 } 338 339 adapter->sc_btag = pa->pa_memt; 340 error = pci_mapreg_info(pa->pa_pc, pa->pa_tag, ENA_REG_BAR, 341 memtype, &adapter->sc_memaddr, &adapter->sc_mapsize, &flags); 342 if (error) { 343 aprint_error_dev(adapter->pdev, "can't get map info\n"); 344 return ENXIO; 345 } 346 347 if (pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &msixoff, 348 NULL)) { 349 pcireg_t msixtbl; 350 bus_size_t table_offset; 351 int bir; 352 353 msixtbl = pci_conf_read(pa->pa_pc, pa->pa_tag, 354 msixoff + PCI_MSIX_TBLOFFSET); 355 table_offset = msixtbl & PCI_MSIX_TBLOFFSET_MASK; 356 bir = msixtbl & PCI_MSIX_TBLBIR_MASK; 357 if (bir == PCI_MAPREG_NUM(ENA_REG_BAR)) 358 adapter->sc_mapsize = table_offset; 359 } 360 361 error = bus_space_map(adapter->sc_btag, adapter->sc_memaddr, 362 adapter->sc_mapsize, flags, &adapter->sc_bhandle); 363 if (error != 0) { 364 aprint_error_dev(adapter->pdev, 365 "can't map mem space (error=%d)\n", error); 366 return ENXIO; 367 } 368 369 return (0); 370 } 371 372 static void 373 ena_free_pci_resources(struct ena_adapter *adapter) 374 { 375 if (adapter->sc_mapsize != 0) { 376 bus_space_unmap(adapter->sc_btag, adapter->sc_bhandle, 377 adapter->sc_mapsize); 378 } 379 } 380 381 static int 382 ena_probe(device_t parent, cfdata_t match, void *aux) 383 { 384 struct pci_attach_args *pa = aux; 385 const ena_vendor_info_t *ent; 386 387 for (int i = 0; i < __arraycount(ena_vendor_info_array); i++) { 388 ent = &ena_vendor_info_array[i]; 389 390 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 391 (PCI_PRODUCT(pa->pa_id) == ent->device_id)) { 392 return 1; 393 } 394 } 395 396 return 0; 397 } 398 399 static int 400 ena_change_mtu(struct ifnet *ifp, int new_mtu) 401 { 402 struct ena_adapter *adapter = if_getsoftc(ifp); 403 int rc; 404 405 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { 406 device_printf(adapter->pdev, "Invalid MTU setting. " 407 "new_mtu: %d max mtu: %d min mtu: %d\n", 408 new_mtu, adapter->max_mtu, ENA_MIN_MTU); 409 return (EINVAL); 410 } 411 412 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); 413 if (likely(rc == 0)) { 414 ena_trace(ENA_DBG, "set MTU to %d\n", new_mtu); 415 if_setmtu(ifp, new_mtu); 416 } else { 417 device_printf(adapter->pdev, "Failed to set MTU to %d\n", 418 new_mtu); 419 } 420 421 return (rc); 422 } 423 424 #define EVCNT_INIT(st, f) \ 425 do { \ 426 evcnt_attach_dynamic(&st->f, EVCNT_TYPE_MISC, NULL, \ 427 st->name, #f); \ 428 } while (0) 429 430 static inline void 431 ena_alloc_counters_rx(struct ena_adapter *adapter, struct ena_stats_rx *st, int queue) 432 { 433 snprintf(st->name, sizeof(st->name), "%s rxq%d", 434 device_xname(adapter->pdev), queue); 435 436 EVCNT_INIT(st, cnt); 437 EVCNT_INIT(st, bytes); 438 EVCNT_INIT(st, refil_partial); 439 EVCNT_INIT(st, bad_csum); 440 EVCNT_INIT(st, mbuf_alloc_fail); 441 EVCNT_INIT(st, dma_mapping_err); 442 EVCNT_INIT(st, bad_desc_num); 443 EVCNT_INIT(st, bad_req_id); 444 EVCNT_INIT(st, empty_rx_ring); 445 446 /* Make sure all code is updated when new fields added */ 447 CTASSERT(offsetof(struct ena_stats_rx, empty_rx_ring) 448 + sizeof(st->empty_rx_ring) == sizeof(*st)); 449 } 450 451 static inline void 452 ena_alloc_counters_tx(struct ena_adapter *adapter, struct ena_stats_tx *st, int queue) 453 { 454 snprintf(st->name, sizeof(st->name), "%s txq%d", 455 device_xname(adapter->pdev), queue); 456 457 EVCNT_INIT(st, cnt); 458 EVCNT_INIT(st, bytes); 459 EVCNT_INIT(st, prepare_ctx_err); 460 EVCNT_INIT(st, dma_mapping_err); 461 EVCNT_INIT(st, doorbells); 462 EVCNT_INIT(st, missing_tx_comp); 463 EVCNT_INIT(st, bad_req_id); 464 EVCNT_INIT(st, collapse); 465 EVCNT_INIT(st, collapse_err); 466 EVCNT_INIT(st, pcq_drops); 467 468 /* Make sure all code is updated when new fields added */ 469 CTASSERT(offsetof(struct ena_stats_tx, pcq_drops) 470 + sizeof(st->pcq_drops) == sizeof(*st)); 471 } 472 473 static inline void 474 ena_alloc_counters_dev(struct ena_adapter *adapter, struct ena_stats_dev *st, int queue) 475 { 476 snprintf(st->name, sizeof(st->name), "%s dev ioq%d", 477 device_xname(adapter->pdev), queue); 478 479 EVCNT_INIT(st, wd_expired); 480 EVCNT_INIT(st, interface_up); 481 EVCNT_INIT(st, interface_down); 482 EVCNT_INIT(st, admin_q_pause); 483 484 /* Make sure all code is updated when new fields added */ 485 CTASSERT(offsetof(struct ena_stats_dev, admin_q_pause) 486 + sizeof(st->admin_q_pause) == sizeof(*st)); 487 } 488 489 static inline void 490 ena_alloc_counters_hwstats(struct ena_adapter *adapter, struct ena_hw_stats *st, int queue) 491 { 492 snprintf(st->name, sizeof(st->name), "%s hw ioq%d", 493 device_xname(adapter->pdev), queue); 494 495 EVCNT_INIT(st, rx_packets); 496 EVCNT_INIT(st, tx_packets); 497 EVCNT_INIT(st, rx_bytes); 498 EVCNT_INIT(st, tx_bytes); 499 EVCNT_INIT(st, rx_drops); 500 501 /* Make sure all code is updated when new fields added */ 502 CTASSERT(offsetof(struct ena_hw_stats, rx_drops) 503 + sizeof(st->rx_drops) == sizeof(*st)); 504 } 505 static inline void 506 ena_free_counters(struct evcnt *begin, int size, int offset) 507 { 508 struct evcnt *end = (struct evcnt *)((char *)begin + size); 509 begin = (struct evcnt *)((char *)begin + offset); 510 511 for (; begin < end; ++begin) 512 counter_u64_free(*begin); 513 } 514 515 static inline void 516 ena_reset_counters(struct evcnt *begin, int size, int offset) 517 { 518 struct evcnt *end = (struct evcnt *)((char *)begin + size); 519 begin = (struct evcnt *)((char *)begin + offset); 520 521 for (; begin < end; ++begin) 522 counter_u64_zero(*begin); 523 } 524 525 static void 526 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, 527 uint16_t qid) 528 { 529 530 ring->qid = qid; 531 ring->adapter = adapter; 532 ring->ena_dev = adapter->ena_dev; 533 } 534 535 static void 536 ena_init_io_rings(struct ena_adapter *adapter) 537 { 538 struct ena_com_dev *ena_dev; 539 struct ena_ring *txr, *rxr; 540 struct ena_que *que; 541 int i; 542 543 ena_dev = adapter->ena_dev; 544 545 for (i = 0; i < adapter->num_queues; i++) { 546 txr = &adapter->tx_ring[i]; 547 rxr = &adapter->rx_ring[i]; 548 549 /* TX/RX common ring state */ 550 ena_init_io_rings_common(adapter, txr, i); 551 ena_init_io_rings_common(adapter, rxr, i); 552 553 /* TX specific ring state */ 554 txr->ring_size = adapter->tx_ring_size; 555 txr->tx_max_header_size = ena_dev->tx_max_header_size; 556 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; 557 txr->smoothed_interval = 558 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev); 559 560 /* Allocate a buf ring */ 561 txr->br = buf_ring_alloc(ena_buf_ring_size, M_DEVBUF, 562 M_WAITOK, &txr->ring_mtx); 563 564 /* Alloc TX statistics. */ 565 ena_alloc_counters_tx(adapter, &txr->tx_stats, i); 566 567 /* RX specific ring state */ 568 rxr->ring_size = adapter->rx_ring_size; 569 rxr->smoothed_interval = 570 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev); 571 572 /* Alloc RX statistics. */ 573 ena_alloc_counters_rx(adapter, &rxr->rx_stats, i); 574 575 /* Initialize locks */ 576 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 577 device_xname(adapter->pdev), i); 578 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 579 device_xname(adapter->pdev), i); 580 581 mutex_init(&txr->ring_mtx, MUTEX_DEFAULT, IPL_NET); 582 mutex_init(&rxr->ring_mtx, MUTEX_DEFAULT, IPL_NET); 583 584 que = &adapter->que[i]; 585 que->adapter = adapter; 586 que->id = i; 587 que->tx_ring = txr; 588 que->rx_ring = rxr; 589 590 txr->que = que; 591 rxr->que = que; 592 593 rxr->empty_rx_queue = 0; 594 } 595 } 596 597 static void 598 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) 599 { 600 struct ena_ring *txr = &adapter->tx_ring[qid]; 601 struct ena_ring *rxr = &adapter->rx_ring[qid]; 602 603 ena_free_counters((struct evcnt *)&txr->tx_stats, 604 sizeof(txr->tx_stats), offsetof(struct ena_stats_tx, cnt)); 605 ena_free_counters((struct evcnt *)&rxr->rx_stats, 606 sizeof(rxr->rx_stats), offsetof(struct ena_stats_rx, cnt)); 607 608 mutex_destroy(&txr->ring_mtx); 609 mutex_destroy(&rxr->ring_mtx); 610 } 611 612 static void 613 ena_free_all_io_rings_resources(struct ena_adapter *adapter) 614 { 615 int i; 616 617 for (i = 0; i < adapter->num_queues; i++) 618 ena_free_io_ring_resources(adapter, i); 619 620 } 621 622 #if 0 623 static int 624 ena_setup_tx_dma_tag(struct ena_adapter *adapter) 625 { 626 int ret; 627 628 /* Create DMA tag for Tx buffers */ 629 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), 630 1, 0, /* alignment, bounds */ 631 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 632 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 633 NULL, NULL, /* filter, filterarg */ 634 ENA_TSO_MAXSIZE, /* maxsize */ 635 adapter->max_tx_sgl_size - 1, /* nsegments */ 636 ENA_TSO_MAXSIZE, /* maxsegsize */ 637 0, /* flags */ 638 NULL, /* lockfunc */ 639 NULL, /* lockfuncarg */ 640 &adapter->tx_buf_tag); 641 642 return (ret); 643 } 644 #endif 645 646 #if 0 647 static int 648 ena_setup_rx_dma_tag(struct ena_adapter *adapter) 649 { 650 int ret; 651 652 /* Create DMA tag for Rx buffers*/ 653 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ 654 1, 0, /* alignment, bounds */ 655 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 656 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 657 NULL, NULL, /* filter, filterarg */ 658 MJUM16BYTES, /* maxsize */ 659 adapter->max_rx_sgl_size, /* nsegments */ 660 MJUM16BYTES, /* maxsegsize */ 661 0, /* flags */ 662 NULL, /* lockfunc */ 663 NULL, /* lockarg */ 664 &adapter->rx_buf_tag); 665 666 return (ret); 667 } 668 #endif 669 670 /** 671 * ena_setup_tx_resources - allocate Tx resources (Descriptors) 672 * @adapter: network interface device structure 673 * @qid: queue index 674 * 675 * Returns 0 on success, otherwise on failure. 676 **/ 677 static int 678 ena_setup_tx_resources(struct ena_adapter *adapter, int qid) 679 { 680 struct ena_que *que = &adapter->que[qid]; 681 struct ena_ring *tx_ring = que->tx_ring; 682 int size, i, err; 683 684 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; 685 tx_ring->tx_buffer_info = kmem_zalloc(size, KM_SLEEP); 686 687 size = sizeof(uint16_t) * tx_ring->ring_size; 688 tx_ring->free_tx_ids = kmem_zalloc(size, KM_SLEEP); 689 690 /* Req id stack for TX OOO completions */ 691 for (i = 0; i < tx_ring->ring_size; i++) 692 tx_ring->free_tx_ids[i] = i; 693 694 /* Reset TX statistics. */ 695 ena_reset_counters((struct evcnt *)&tx_ring->tx_stats, 696 sizeof(tx_ring->tx_stats), 697 offsetof(struct ena_stats_tx, cnt)); 698 699 tx_ring->next_to_use = 0; 700 tx_ring->next_to_clean = 0; 701 702 tx_ring->br = pcq_create(ENA_DEFAULT_RING_SIZE, KM_SLEEP); 703 704 /* ... and create the buffer DMA maps */ 705 for (i = 0; i < tx_ring->ring_size; i++) { 706 err = bus_dmamap_create(adapter->sc_dmat, 707 ENA_TSO_MAXSIZE, adapter->max_tx_sgl_size - 1, 708 ENA_TSO_MAXSIZE, 0, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 709 &tx_ring->tx_buffer_info[i].map); 710 if (unlikely(err != 0)) { 711 ena_trace(ENA_ALERT, 712 "Unable to create Tx DMA map for buffer %d\n", i); 713 goto err_buf_info_unmap; 714 } 715 } 716 717 /* Allocate workqueues */ 718 int rc = workqueue_create(&tx_ring->enqueue_tq, "ena_tx_enq", 719 ena_deferred_mq_start, tx_ring, 0, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 720 if (unlikely(rc != 0)) { 721 ena_trace(ENA_ALERT, 722 "Unable to create workqueue for enqueue task\n"); 723 i = tx_ring->ring_size; 724 goto err_buf_info_unmap; 725 } 726 return (0); 727 728 err_buf_info_unmap: 729 while (i--) { 730 bus_dmamap_destroy(adapter->sc_dmat, 731 tx_ring->tx_buffer_info[i].map); 732 } 733 size = sizeof(uint16_t) * tx_ring->ring_size; 734 kmem_free(tx_ring->free_tx_ids, size); 735 tx_ring->free_tx_ids = NULL; 736 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; 737 kmem_free(tx_ring->tx_buffer_info, size); 738 tx_ring->tx_buffer_info = NULL; 739 740 return (ENOMEM); 741 } 742 743 /** 744 * ena_free_tx_resources - Free Tx Resources per Queue 745 * @adapter: network interface device structure 746 * @qid: queue index 747 * 748 * Free all transmit software resources 749 **/ 750 static void 751 ena_free_tx_resources(struct ena_adapter *adapter, int qid) 752 { 753 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 754 struct mbuf *m; 755 756 workqueue_wait(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 757 workqueue_destroy(tx_ring->enqueue_tq); 758 tx_ring->enqueue_tq = NULL; 759 760 /* Flush buffer ring, */ 761 while ((m = pcq_get(tx_ring->br)) != NULL) 762 m_freem(m); 763 pcq_destroy(tx_ring->br); 764 tx_ring->br = NULL; 765 766 /* Free buffer DMA maps, */ 767 for (int i = 0; i < tx_ring->ring_size; i++) { 768 bus_dmamap_unload(adapter->sc_dmat, 769 tx_ring->tx_buffer_info[i].map); 770 bus_dmamap_destroy(adapter->sc_dmat, 771 tx_ring->tx_buffer_info[i].map); 772 m_freem(tx_ring->tx_buffer_info[i].mbuf); 773 tx_ring->tx_buffer_info[i].mbuf = NULL; 774 } 775 776 /* And free allocated memory. */ 777 kmem_free(tx_ring->tx_buffer_info, 778 sizeof(struct ena_tx_buffer) * tx_ring->ring_size); 779 tx_ring->tx_buffer_info = NULL; 780 781 kmem_free(tx_ring->free_tx_ids, sizeof(uint16_t) * tx_ring->ring_size); 782 tx_ring->free_tx_ids = NULL; 783 } 784 785 /** 786 * ena_setup_all_tx_resources - allocate all queues Tx resources 787 * @adapter: network interface device structure 788 * 789 * Returns 0 on success, otherwise on failure. 790 **/ 791 static int 792 ena_setup_all_tx_resources(struct ena_adapter *adapter) 793 { 794 int i, rc; 795 796 for (i = 0; i < adapter->num_queues; i++) { 797 rc = ena_setup_tx_resources(adapter, i); 798 if (rc != 0) { 799 device_printf(adapter->pdev, 800 "Allocation for Tx Queue %u failed\n", i); 801 goto err_setup_tx; 802 } 803 } 804 805 return (0); 806 807 err_setup_tx: 808 /* Rewind the index freeing the rings as we go */ 809 while (i--) 810 ena_free_tx_resources(adapter, i); 811 return (rc); 812 } 813 814 /** 815 * ena_free_all_tx_resources - Free Tx Resources for All Queues 816 * @adapter: network interface device structure 817 * 818 * Free all transmit software resources 819 **/ 820 static void 821 ena_free_all_tx_resources(struct ena_adapter *adapter) 822 { 823 int i; 824 825 for (i = 0; i < adapter->num_queues; i++) 826 ena_free_tx_resources(adapter, i); 827 } 828 829 static inline int 830 validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id) 831 { 832 if (likely(req_id < rx_ring->ring_size)) 833 return (0); 834 835 device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n", 836 req_id); 837 counter_u64_add(rx_ring->rx_stats.bad_req_id, 1); 838 839 /* Trigger device reset */ 840 rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; 841 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter); 842 843 return (EFAULT); 844 } 845 846 /** 847 * ena_setup_rx_resources - allocate Rx resources (Descriptors) 848 * @adapter: network interface device structure 849 * @qid: queue index 850 * 851 * Returns 0 on success, otherwise on failure. 852 **/ 853 static int 854 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) 855 { 856 struct ena_que *que = &adapter->que[qid]; 857 struct ena_ring *rx_ring = que->rx_ring; 858 int size, err, i; 859 860 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; 861 862 /* 863 * Alloc extra element so in rx path 864 * we can always prefetch rx_info + 1 865 */ 866 size += sizeof(struct ena_rx_buffer); 867 868 rx_ring->rx_buffer_info = kmem_zalloc(size, KM_SLEEP); 869 870 size = sizeof(uint16_t) * rx_ring->ring_size; 871 rx_ring->free_rx_ids = kmem_zalloc(size, KM_SLEEP); 872 873 for (i = 0; i < rx_ring->ring_size; i++) 874 rx_ring->free_rx_ids[i] = i; 875 876 /* Reset RX statistics. */ 877 ena_reset_counters((struct evcnt *)&rx_ring->rx_stats, 878 sizeof(rx_ring->rx_stats), 879 offsetof(struct ena_stats_rx, cnt)); 880 881 rx_ring->next_to_clean = 0; 882 rx_ring->next_to_use = 0; 883 884 /* ... and create the buffer DMA maps */ 885 for (i = 0; i < rx_ring->ring_size; i++) { 886 err = bus_dmamap_create(adapter->sc_dmat, 887 MJUM16BYTES, adapter->max_rx_sgl_size, MJUM16BYTES, 888 0, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 889 &(rx_ring->rx_buffer_info[i].map)); 890 if (err != 0) { 891 ena_trace(ENA_ALERT, 892 "Unable to create Rx DMA map for buffer %d\n", i); 893 goto err_buf_info_unmap; 894 } 895 } 896 897 #ifdef LRO 898 /* Create LRO for the ring */ 899 if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) { 900 int err = tcp_lro_init(&rx_ring->lro); 901 if (err != 0) { 902 device_printf(adapter->pdev, 903 "LRO[%d] Initialization failed!\n", qid); 904 } else { 905 ena_trace(ENA_INFO, 906 "RX Soft LRO[%d] Initialized\n", qid); 907 rx_ring->lro.ifp = adapter->ifp; 908 } 909 } 910 #endif 911 912 /* Allocate workqueues */ 913 int rc = workqueue_create(&rx_ring->cleanup_tq, "ena_rx_comp", 914 ena_cleanup, que, 0, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 915 if (unlikely(rc != 0)) { 916 ena_trace(ENA_ALERT, 917 "Unable to create workqueue for RX completion task\n"); 918 goto err_buf_info_unmap; 919 } 920 921 return (0); 922 923 err_buf_info_unmap: 924 while (i--) { 925 bus_dmamap_destroy(adapter->sc_dmat, 926 rx_ring->rx_buffer_info[i].map); 927 } 928 929 size = sizeof(uint16_t) * rx_ring->ring_size; 930 kmem_free(rx_ring->free_rx_ids, size); 931 rx_ring->free_rx_ids = NULL; 932 size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1); 933 kmem_free(rx_ring->rx_buffer_info, size); 934 rx_ring->rx_buffer_info = NULL; 935 return (ENOMEM); 936 } 937 938 /** 939 * ena_free_rx_resources - Free Rx Resources 940 * @adapter: network interface device structure 941 * @qid: queue index 942 * 943 * Free all receive software resources 944 **/ 945 static void 946 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid) 947 { 948 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 949 950 workqueue_wait(rx_ring->cleanup_tq, &rx_ring->cleanup_task); 951 workqueue_destroy(rx_ring->cleanup_tq); 952 rx_ring->cleanup_tq = NULL; 953 954 /* Free buffer DMA maps, */ 955 for (int i = 0; i < rx_ring->ring_size; i++) { 956 bus_dmamap_unload(adapter->sc_dmat, 957 rx_ring->rx_buffer_info[i].map); 958 bus_dmamap_destroy(adapter->sc_dmat, 959 rx_ring->rx_buffer_info[i].map); 960 m_freem(rx_ring->rx_buffer_info[i].mbuf); 961 rx_ring->rx_buffer_info[i].mbuf = NULL; 962 } 963 964 #ifdef LRO 965 /* free LRO resources, */ 966 tcp_lro_free(&rx_ring->lro); 967 #endif 968 969 /* free allocated memory */ 970 kmem_free(rx_ring->rx_buffer_info, 971 sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1)); 972 rx_ring->rx_buffer_info = NULL; 973 974 kmem_free(rx_ring->free_rx_ids, sizeof(uint16_t) * rx_ring->ring_size); 975 rx_ring->free_rx_ids = NULL; 976 } 977 978 /** 979 * ena_setup_all_rx_resources - allocate all queues Rx resources 980 * @adapter: network interface device structure 981 * 982 * Returns 0 on success, otherwise on failure. 983 **/ 984 static int 985 ena_setup_all_rx_resources(struct ena_adapter *adapter) 986 { 987 int i, rc = 0; 988 989 for (i = 0; i < adapter->num_queues; i++) { 990 rc = ena_setup_rx_resources(adapter, i); 991 if (rc != 0) { 992 device_printf(adapter->pdev, 993 "Allocation for Rx Queue %u failed\n", i); 994 goto err_setup_rx; 995 } 996 } 997 return (0); 998 999 err_setup_rx: 1000 /* rewind the index freeing the rings as we go */ 1001 while (i--) 1002 ena_free_rx_resources(adapter, i); 1003 return (rc); 1004 } 1005 1006 /** 1007 * ena_free_all_rx_resources - Free Rx resources for all queues 1008 * @adapter: network interface device structure 1009 * 1010 * Free all receive software resources 1011 **/ 1012 static void 1013 ena_free_all_rx_resources(struct ena_adapter *adapter) 1014 { 1015 int i; 1016 1017 for (i = 0; i < adapter->num_queues; i++) 1018 ena_free_rx_resources(adapter, i); 1019 } 1020 1021 static inline int 1022 ena_alloc_rx_mbuf(struct ena_adapter *adapter, 1023 struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) 1024 { 1025 struct ena_com_buf *ena_buf; 1026 int error; 1027 int mlen; 1028 1029 /* if previous allocated frag is not used */ 1030 if (unlikely(rx_info->mbuf != NULL)) 1031 return (0); 1032 1033 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1034 if (unlikely(rx_info->mbuf == NULL)) { 1035 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 1036 return (ENOMEM); 1037 } 1038 mlen = MCLBYTES; 1039 1040 /* Set mbuf length*/ 1041 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen; 1042 1043 /* Map packets for DMA */ 1044 ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH, 1045 "Using tag %p for buffers' DMA mapping, mbuf %p len: %d", 1046 adapter->sc_dmat,rx_info->mbuf, rx_info->mbuf->m_len); 1047 error = bus_dmamap_load_mbuf(adapter->sc_dmat, rx_info->map, 1048 rx_info->mbuf, BUS_DMA_NOWAIT); 1049 if (unlikely((error != 0) || (rx_info->map->dm_nsegs != 1))) { 1050 ena_trace(ENA_WARNING, "failed to map mbuf, error: %d, " 1051 "nsegs: %d\n", error, rx_info->map->dm_nsegs); 1052 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1); 1053 goto exit; 1054 1055 } 1056 1057 bus_dmamap_sync(adapter->sc_dmat, rx_info->map, 0, 1058 rx_info->map->dm_mapsize, BUS_DMASYNC_PREREAD); 1059 1060 ena_buf = &rx_info->ena_buf; 1061 ena_buf->paddr = rx_info->map->dm_segs[0].ds_addr; 1062 ena_buf->len = mlen; 1063 1064 ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH, 1065 "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n", 1066 rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr); 1067 1068 return (0); 1069 1070 exit: 1071 m_freem(rx_info->mbuf); 1072 rx_info->mbuf = NULL; 1073 return (EFAULT); 1074 } 1075 1076 static void 1077 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, 1078 struct ena_rx_buffer *rx_info) 1079 { 1080 1081 if (rx_info->mbuf == NULL) { 1082 ena_trace(ENA_WARNING, "Trying to free unallocated buffer\n"); 1083 return; 1084 } 1085 1086 bus_dmamap_unload(adapter->sc_dmat, rx_info->map); 1087 m_freem(rx_info->mbuf); 1088 rx_info->mbuf = NULL; 1089 } 1090 1091 /** 1092 * ena_refill_rx_bufs - Refills ring with descriptors 1093 * @rx_ring: the ring which we want to feed with free descriptors 1094 * @num: number of descriptors to refill 1095 * Refills the ring with newly allocated DMA-mapped mbufs for receiving 1096 **/ 1097 static int 1098 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num) 1099 { 1100 struct ena_adapter *adapter = rx_ring->adapter; 1101 uint16_t next_to_use, req_id; 1102 uint32_t i; 1103 int rc; 1104 1105 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d", 1106 rx_ring->qid); 1107 1108 next_to_use = rx_ring->next_to_use; 1109 1110 for (i = 0; i < num; i++) { 1111 struct ena_rx_buffer *rx_info; 1112 1113 ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, 1114 "RX buffer - next to use: %d", next_to_use); 1115 1116 req_id = rx_ring->free_rx_ids[next_to_use]; 1117 rc = validate_rx_req_id(rx_ring, req_id); 1118 if (unlikely(rc != 0)) 1119 break; 1120 1121 rx_info = &rx_ring->rx_buffer_info[req_id]; 1122 1123 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); 1124 if (unlikely(rc != 0)) { 1125 ena_trace(ENA_WARNING, 1126 "failed to alloc buffer for rx queue %d\n", 1127 rx_ring->qid); 1128 break; 1129 } 1130 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, 1131 &rx_info->ena_buf, req_id); 1132 if (unlikely(rc != 0)) { 1133 ena_trace(ENA_WARNING, 1134 "failed to add buffer for rx queue %d\n", 1135 rx_ring->qid); 1136 break; 1137 } 1138 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, 1139 rx_ring->ring_size); 1140 } 1141 1142 if (unlikely(i < num)) { 1143 counter_u64_add(rx_ring->rx_stats.refil_partial, 1); 1144 ena_trace(ENA_WARNING, 1145 "refilled rx qid %d with only %d mbufs (from %d)\n", 1146 rx_ring->qid, i, num); 1147 } 1148 1149 if (likely(i != 0)) { 1150 wmb(); 1151 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); 1152 } 1153 rx_ring->next_to_use = next_to_use; 1154 return (i); 1155 } 1156 1157 static void 1158 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid) 1159 { 1160 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 1161 unsigned int i; 1162 1163 for (i = 0; i < rx_ring->ring_size; i++) { 1164 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; 1165 1166 if (rx_info->mbuf != NULL) 1167 ena_free_rx_mbuf(adapter, rx_ring, rx_info); 1168 } 1169 } 1170 1171 /** 1172 * ena_refill_all_rx_bufs - allocate all queues Rx buffers 1173 * @adapter: network interface device structure 1174 * 1175 */ 1176 static void 1177 ena_refill_all_rx_bufs(struct ena_adapter *adapter) 1178 { 1179 struct ena_ring *rx_ring; 1180 int i, rc, bufs_num; 1181 1182 for (i = 0; i < adapter->num_queues; i++) { 1183 rx_ring = &adapter->rx_ring[i]; 1184 bufs_num = rx_ring->ring_size - 1; 1185 rc = ena_refill_rx_bufs(rx_ring, bufs_num); 1186 1187 if (unlikely(rc != bufs_num)) 1188 ena_trace(ENA_WARNING, "refilling Queue %d failed. " 1189 "Allocated %d buffers from: %d\n", i, rc, bufs_num); 1190 } 1191 } 1192 1193 static void 1194 ena_free_all_rx_bufs(struct ena_adapter *adapter) 1195 { 1196 int i; 1197 1198 for (i = 0; i < adapter->num_queues; i++) 1199 ena_free_rx_bufs(adapter, i); 1200 } 1201 1202 /** 1203 * ena_free_tx_bufs - Free Tx Buffers per Queue 1204 * @adapter: network interface device structure 1205 * @qid: queue index 1206 **/ 1207 static void 1208 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid) 1209 { 1210 bool print_once = true; 1211 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 1212 1213 for (int i = 0; i < tx_ring->ring_size; i++) { 1214 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; 1215 1216 if (tx_info->mbuf == NULL) 1217 continue; 1218 1219 if (print_once) { 1220 device_printf(adapter->pdev, 1221 "free uncompleted tx mbuf qid %d idx 0x%x", 1222 qid, i); 1223 print_once = false; 1224 } else { 1225 ena_trace(ENA_DBG, 1226 "free uncompleted tx mbuf qid %d idx 0x%x", 1227 qid, i); 1228 } 1229 1230 bus_dmamap_unload(adapter->sc_dmat, tx_info->map); 1231 m_free(tx_info->mbuf); 1232 tx_info->mbuf = NULL; 1233 } 1234 } 1235 1236 static void 1237 ena_free_all_tx_bufs(struct ena_adapter *adapter) 1238 { 1239 1240 for (int i = 0; i < adapter->num_queues; i++) 1241 ena_free_tx_bufs(adapter, i); 1242 } 1243 1244 static void 1245 ena_destroy_all_tx_queues(struct ena_adapter *adapter) 1246 { 1247 uint16_t ena_qid; 1248 int i; 1249 1250 for (i = 0; i < adapter->num_queues; i++) { 1251 ena_qid = ENA_IO_TXQ_IDX(i); 1252 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1253 } 1254 } 1255 1256 static void 1257 ena_destroy_all_rx_queues(struct ena_adapter *adapter) 1258 { 1259 uint16_t ena_qid; 1260 int i; 1261 1262 for (i = 0; i < adapter->num_queues; i++) { 1263 ena_qid = ENA_IO_RXQ_IDX(i); 1264 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1265 } 1266 } 1267 1268 static void 1269 ena_destroy_all_io_queues(struct ena_adapter *adapter) 1270 { 1271 ena_destroy_all_tx_queues(adapter); 1272 ena_destroy_all_rx_queues(adapter); 1273 } 1274 1275 static inline int 1276 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id) 1277 { 1278 struct ena_adapter *adapter = tx_ring->adapter; 1279 struct ena_tx_buffer *tx_info = NULL; 1280 KASSERT(ENA_RING_MTX_OWNED(tx_ring)); 1281 1282 if (likely(req_id < tx_ring->ring_size)) { 1283 tx_info = &tx_ring->tx_buffer_info[req_id]; 1284 if (tx_info->mbuf != NULL) 1285 return (0); 1286 } 1287 1288 if (tx_info->mbuf == NULL) 1289 device_printf(adapter->pdev, 1290 "tx_info doesn't have valid mbuf\n"); 1291 else 1292 device_printf(adapter->pdev, "Invalid req_id: %hu\n", req_id); 1293 1294 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1); 1295 1296 return (EFAULT); 1297 } 1298 1299 static int 1300 ena_create_io_queues(struct ena_adapter *adapter) 1301 { 1302 struct ena_com_dev *ena_dev = adapter->ena_dev; 1303 struct ena_com_create_io_ctx ctx; 1304 struct ena_ring *ring; 1305 uint16_t ena_qid; 1306 uint32_t msix_vector; 1307 int rc, i; 1308 1309 /* Create TX queues */ 1310 for (i = 0; i < adapter->num_queues; i++) { 1311 msix_vector = ENA_IO_IRQ_IDX(i); 1312 ena_qid = ENA_IO_TXQ_IDX(i); 1313 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1314 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1315 ctx.queue_size = adapter->tx_ring_size; 1316 ctx.msix_vector = msix_vector; 1317 ctx.qid = ena_qid; 1318 rc = ena_com_create_io_queue(ena_dev, &ctx); 1319 if (rc != 0) { 1320 device_printf(adapter->pdev, 1321 "Failed to create io TX queue #%d rc: %d\n", i, rc); 1322 goto err_tx; 1323 } 1324 ring = &adapter->tx_ring[i]; 1325 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1326 &ring->ena_com_io_sq, 1327 &ring->ena_com_io_cq); 1328 if (rc != 0) { 1329 device_printf(adapter->pdev, 1330 "Failed to get TX queue handlers. TX queue num" 1331 " %d rc: %d\n", i, rc); 1332 ena_com_destroy_io_queue(ena_dev, ena_qid); 1333 goto err_tx; 1334 } 1335 } 1336 1337 /* Create RX queues */ 1338 for (i = 0; i < adapter->num_queues; i++) { 1339 msix_vector = ENA_IO_IRQ_IDX(i); 1340 ena_qid = ENA_IO_RXQ_IDX(i); 1341 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1342 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1343 ctx.queue_size = adapter->rx_ring_size; 1344 ctx.msix_vector = msix_vector; 1345 ctx.qid = ena_qid; 1346 rc = ena_com_create_io_queue(ena_dev, &ctx); 1347 if (unlikely(rc != 0)) { 1348 device_printf(adapter->pdev, 1349 "Failed to create io RX queue[%d] rc: %d\n", i, rc); 1350 goto err_rx; 1351 } 1352 1353 ring = &adapter->rx_ring[i]; 1354 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1355 &ring->ena_com_io_sq, 1356 &ring->ena_com_io_cq); 1357 if (unlikely(rc != 0)) { 1358 device_printf(adapter->pdev, 1359 "Failed to get RX queue handlers. RX queue num" 1360 " %d rc: %d\n", i, rc); 1361 ena_com_destroy_io_queue(ena_dev, ena_qid); 1362 goto err_rx; 1363 } 1364 } 1365 1366 return (0); 1367 1368 err_rx: 1369 while (i--) 1370 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i)); 1371 i = adapter->num_queues; 1372 err_tx: 1373 while (i--) 1374 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i)); 1375 1376 return (ENXIO); 1377 } 1378 1379 /** 1380 * ena_tx_cleanup - clear sent packets and corresponding descriptors 1381 * @tx_ring: ring for which we want to clean packets 1382 * 1383 * Once packets are sent, we ask the device in a loop for no longer used 1384 * descriptors. We find the related mbuf chain in a map (index in an array) 1385 * and free it, then update ring state. 1386 * This is performed in "endless" loop, updating ring pointers every 1387 * TX_COMMIT. The first check of free descriptor is performed before the actual 1388 * loop, then repeated at the loop end. 1389 **/ 1390 static int 1391 ena_tx_cleanup(struct ena_ring *tx_ring) 1392 { 1393 struct ena_adapter *adapter; 1394 struct ena_com_io_cq* io_cq; 1395 uint16_t next_to_clean; 1396 uint16_t req_id; 1397 uint16_t ena_qid; 1398 unsigned int total_done = 0; 1399 int rc; 1400 int commit = TX_COMMIT; 1401 int budget = TX_BUDGET; 1402 int work_done; 1403 1404 KASSERT(ENA_RING_MTX_OWNED(tx_ring)); 1405 1406 adapter = tx_ring->que->adapter; 1407 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 1408 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 1409 next_to_clean = tx_ring->next_to_clean; 1410 1411 do { 1412 struct ena_tx_buffer *tx_info; 1413 struct mbuf *mbuf; 1414 1415 rc = ena_com_tx_comp_req_id_get(io_cq, &req_id); 1416 if (unlikely(rc != 0)) 1417 break; 1418 1419 rc = validate_tx_req_id(tx_ring, req_id); 1420 if (unlikely(rc != 0)) 1421 break; 1422 1423 tx_info = &tx_ring->tx_buffer_info[req_id]; 1424 1425 mbuf = tx_info->mbuf; 1426 1427 tx_info->mbuf = NULL; 1428 bintime_clear(&tx_info->timestamp); 1429 1430 if (likely(tx_info->num_of_bufs != 0)) { 1431 /* Map is no longer required */ 1432 bus_dmamap_unload(adapter->sc_dmat, tx_info->map); 1433 } 1434 1435 ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed", 1436 tx_ring->qid, mbuf); 1437 1438 m_freem(mbuf); 1439 1440 total_done += tx_info->tx_descs; 1441 1442 tx_ring->free_tx_ids[next_to_clean] = req_id; 1443 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean, 1444 tx_ring->ring_size); 1445 1446 if (unlikely(--commit == 0)) { 1447 commit = TX_COMMIT; 1448 /* update ring state every TX_COMMIT descriptor */ 1449 tx_ring->next_to_clean = next_to_clean; 1450 ena_com_comp_ack( 1451 &adapter->ena_dev->io_sq_queues[ena_qid], 1452 total_done); 1453 ena_com_update_dev_comp_head(io_cq); 1454 total_done = 0; 1455 } 1456 } while (likely(--budget)); 1457 1458 work_done = TX_BUDGET - budget; 1459 1460 ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d", 1461 tx_ring->qid, work_done); 1462 1463 /* If there is still something to commit update ring state */ 1464 if (likely(commit != TX_COMMIT)) { 1465 tx_ring->next_to_clean = next_to_clean; 1466 ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid], 1467 total_done); 1468 ena_com_update_dev_comp_head(io_cq); 1469 } 1470 1471 if (atomic_cas_uint(&tx_ring->task_pending, 0, 1) == 0) 1472 workqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task, NULL); 1473 1474 return (work_done); 1475 } 1476 1477 #if 0 1478 static void 1479 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 1480 struct mbuf *mbuf) 1481 { 1482 struct ena_adapter *adapter = rx_ring->adapter; 1483 1484 if (likely(adapter->rss_support)) { 1485 mbuf->m_pkthdr.flowid = ena_rx_ctx->hash; 1486 1487 if (ena_rx_ctx->frag && 1488 (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) { 1489 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 1490 return; 1491 } 1492 1493 switch (ena_rx_ctx->l3_proto) { 1494 case ENA_ETH_IO_L3_PROTO_IPV4: 1495 switch (ena_rx_ctx->l4_proto) { 1496 case ENA_ETH_IO_L4_PROTO_TCP: 1497 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4); 1498 break; 1499 case ENA_ETH_IO_L4_PROTO_UDP: 1500 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4); 1501 break; 1502 default: 1503 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4); 1504 } 1505 break; 1506 case ENA_ETH_IO_L3_PROTO_IPV6: 1507 switch (ena_rx_ctx->l4_proto) { 1508 case ENA_ETH_IO_L4_PROTO_TCP: 1509 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6); 1510 break; 1511 case ENA_ETH_IO_L4_PROTO_UDP: 1512 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6); 1513 break; 1514 default: 1515 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6); 1516 } 1517 break; 1518 case ENA_ETH_IO_L3_PROTO_UNKNOWN: 1519 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 1520 break; 1521 default: 1522 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 1523 } 1524 } else { 1525 mbuf->m_pkthdr.flowid = rx_ring->qid; 1526 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 1527 } 1528 } 1529 #endif 1530 1531 /** 1532 * ena_rx_mbuf - assemble mbuf from descriptors 1533 * @rx_ring: ring for which we want to clean packets 1534 * @ena_bufs: buffer info 1535 * @ena_rx_ctx: metadata for this packet(s) 1536 * @next_to_clean: ring pointer, will be updated only upon success 1537 * 1538 **/ 1539 static struct mbuf* 1540 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs, 1541 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean) 1542 { 1543 struct mbuf *mbuf; 1544 struct ena_rx_buffer *rx_info; 1545 struct ena_adapter *adapter; 1546 unsigned int descs = ena_rx_ctx->descs; 1547 uint16_t ntc, len, req_id, buf = 0; 1548 1549 ntc = *next_to_clean; 1550 adapter = rx_ring->adapter; 1551 1552 len = ena_bufs[buf].len; 1553 req_id = ena_bufs[buf].req_id; 1554 rx_info = &rx_ring->rx_buffer_info[req_id]; 1555 if (unlikely(rx_info->mbuf == NULL)) { 1556 device_printf(adapter->pdev, "NULL mbuf in rx_info"); 1557 return (NULL); 1558 } 1559 1560 ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx", 1561 rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr); 1562 1563 mbuf = rx_info->mbuf; 1564 KASSERT(mbuf->m_flags & M_PKTHDR); 1565 mbuf->m_pkthdr.len = len; 1566 mbuf->m_len = len; 1567 m_set_rcvif(mbuf, rx_ring->que->adapter->ifp); 1568 1569 /* Fill mbuf with hash key and it's interpretation for optimization */ 1570 #if 0 1571 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf); 1572 #endif 1573 1574 ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf %p, flags=0x%x, len: %d", 1575 mbuf, mbuf->m_flags, mbuf->m_pkthdr.len); 1576 1577 /* DMA address is not needed anymore, unmap it */ 1578 bus_dmamap_unload(rx_ring->adapter->sc_dmat, rx_info->map); 1579 1580 rx_info->mbuf = NULL; 1581 rx_ring->free_rx_ids[ntc] = req_id; 1582 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 1583 1584 /* 1585 * While we have more than 1 descriptors for one rcvd packet, append 1586 * other mbufs to the main one 1587 */ 1588 while (--descs) { 1589 ++buf; 1590 len = ena_bufs[buf].len; 1591 req_id = ena_bufs[buf].req_id; 1592 rx_info = &rx_ring->rx_buffer_info[req_id]; 1593 1594 if (unlikely(rx_info->mbuf == NULL)) { 1595 device_printf(adapter->pdev, "NULL mbuf in rx_info"); 1596 /* 1597 * If one of the required mbufs was not allocated yet, 1598 * we can break there. 1599 * All earlier used descriptors will be reallocated 1600 * later and not used mbufs can be reused. 1601 * The next_to_clean pointer will not be updated in case 1602 * of an error, so caller should advance it manually 1603 * in error handling routine to keep it up to date 1604 * with hw ring. 1605 */ 1606 m_freem(mbuf); 1607 return (NULL); 1608 } 1609 1610 if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) { 1611 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 1612 ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p", 1613 mbuf); 1614 } 1615 1616 ena_trace(ENA_DBG | ENA_RXPTH, 1617 "rx mbuf updated. len %d", mbuf->m_pkthdr.len); 1618 1619 /* Free already appended mbuf, it won't be useful anymore */ 1620 bus_dmamap_unload(rx_ring->adapter->sc_dmat, rx_info->map); 1621 m_freem(rx_info->mbuf); 1622 rx_info->mbuf = NULL; 1623 1624 rx_ring->free_rx_ids[ntc] = req_id; 1625 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size); 1626 } 1627 1628 *next_to_clean = ntc; 1629 1630 return (mbuf); 1631 } 1632 1633 /** 1634 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum 1635 **/ 1636 static inline void 1637 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx, 1638 struct mbuf *mbuf) 1639 { 1640 1641 /* IPv4 */ 1642 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) { 1643 mbuf->m_pkthdr.csum_flags |= M_CSUM_IPv4; 1644 if (ena_rx_ctx->l3_csum_err) { 1645 /* ipv4 checksum error */ 1646 mbuf->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD; 1647 counter_u64_add(rx_ring->rx_stats.bad_csum, 1); 1648 ena_trace(ENA_DBG, "RX IPv4 header checksum error"); 1649 return; 1650 } 1651 1652 /* TCP/UDP */ 1653 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || 1654 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) { 1655 mbuf->m_pkthdr.csum_flags |= (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ? M_CSUM_TCPv4 : M_CSUM_UDPv4; 1656 if (ena_rx_ctx->l4_csum_err) { 1657 /* TCP/UDP checksum error */ 1658 mbuf->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 1659 counter_u64_add(rx_ring->rx_stats.bad_csum, 1); 1660 ena_trace(ENA_DBG, "RX L4 checksum error"); 1661 } 1662 } 1663 } 1664 /* IPv6 */ 1665 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) { 1666 /* TCP/UDP */ 1667 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) || 1668 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) { 1669 mbuf->m_pkthdr.csum_flags |= (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ? M_CSUM_TCPv6 : M_CSUM_UDPv6; 1670 if (ena_rx_ctx->l4_csum_err) { 1671 /* TCP/UDP checksum error */ 1672 mbuf->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD; 1673 counter_u64_add(rx_ring->rx_stats.bad_csum, 1); 1674 ena_trace(ENA_DBG, "RX L4 checksum error"); 1675 } 1676 } 1677 } 1678 } 1679 1680 /** 1681 * ena_rx_cleanup - handle rx irq 1682 * @arg: ring for which irq is being handled 1683 **/ 1684 static int 1685 ena_rx_cleanup(struct ena_ring *rx_ring) 1686 { 1687 struct ena_adapter *adapter; 1688 struct mbuf *mbuf; 1689 struct ena_com_rx_ctx ena_rx_ctx; 1690 struct ena_com_io_cq* io_cq; 1691 struct ena_com_io_sq* io_sq; 1692 struct ifnet *ifp; 1693 uint16_t ena_qid; 1694 uint16_t next_to_clean; 1695 uint32_t refill_required; 1696 uint32_t refill_threshold; 1697 uint32_t do_if_input = 0; 1698 unsigned int qid; 1699 int rc, i; 1700 int budget = RX_BUDGET; 1701 1702 adapter = rx_ring->que->adapter; 1703 ifp = adapter->ifp; 1704 qid = rx_ring->que->id; 1705 ena_qid = ENA_IO_RXQ_IDX(qid); 1706 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 1707 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; 1708 next_to_clean = rx_ring->next_to_clean; 1709 1710 ena_trace(ENA_DBG, "rx: qid %d", qid); 1711 1712 do { 1713 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1714 ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size; 1715 ena_rx_ctx.descs = 0; 1716 rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx); 1717 1718 if (unlikely(rc != 0)) 1719 goto error; 1720 1721 if (unlikely(ena_rx_ctx.descs == 0)) 1722 break; 1723 1724 ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. " 1725 "descs #: %d l3 proto %d l4 proto %d hash: %x", 1726 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto, 1727 ena_rx_ctx.l4_proto, ena_rx_ctx.hash); 1728 1729 /* Receive mbuf from the ring */ 1730 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, 1731 &ena_rx_ctx, &next_to_clean); 1732 1733 /* Exit if we failed to retrieve a buffer */ 1734 if (unlikely(mbuf == NULL)) { 1735 for (i = 0; i < ena_rx_ctx.descs; ++i) { 1736 rx_ring->free_rx_ids[next_to_clean] = 1737 rx_ring->ena_bufs[i].req_id; 1738 next_to_clean = 1739 ENA_RX_RING_IDX_NEXT(next_to_clean, 1740 rx_ring->ring_size); 1741 1742 } 1743 if_statinc(ifp, if_ierrors); 1744 break; 1745 } 1746 1747 if (((ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) != 0) || 1748 ((ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx) != 0) || 1749 ((ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx) != 0) || 1750 ((ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx) != 0) || 1751 ((ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx) != 0)) { 1752 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf); 1753 } 1754 1755 counter_enter(); 1756 counter_u64_add_protected(rx_ring->rx_stats.bytes, 1757 mbuf->m_pkthdr.len); 1758 counter_u64_add_protected(adapter->hw_stats.rx_bytes, 1759 mbuf->m_pkthdr.len); 1760 counter_exit(); 1761 /* 1762 * LRO is only for IP/TCP packets and TCP checksum of the packet 1763 * should be computed by hardware. 1764 */ 1765 do_if_input = 1; 1766 #ifdef LRO 1767 if (((ifp->if_capenable & IFCAP_LRO) != 0) && 1768 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) && 1769 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) { 1770 /* 1771 * Send to the stack if: 1772 * - LRO not enabled, or 1773 * - no LRO resources, or 1774 * - lro enqueue fails 1775 */ 1776 if ((rx_ring->lro.lro_cnt != 0) && 1777 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0)) 1778 do_if_input = 0; 1779 } 1780 #endif 1781 if (do_if_input != 0) { 1782 ena_trace(ENA_DBG | ENA_RXPTH, 1783 "calling if_input() with mbuf %p", mbuf); 1784 if_percpuq_enqueue(ifp->if_percpuq, mbuf); 1785 } 1786 1787 counter_enter(); 1788 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1); 1789 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1); 1790 counter_exit(); 1791 } while (--budget); 1792 1793 rx_ring->next_to_clean = next_to_clean; 1794 1795 refill_required = ena_com_free_desc(io_sq); 1796 refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER; 1797 1798 if (refill_required > refill_threshold) { 1799 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); 1800 ena_refill_rx_bufs(rx_ring, refill_required); 1801 } 1802 1803 #ifdef LRO 1804 tcp_lro_flush_all(&rx_ring->lro); 1805 #endif 1806 1807 return (RX_BUDGET - budget); 1808 1809 error: 1810 counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1); 1811 return (RX_BUDGET - budget); 1812 } 1813 1814 /********************************************************************* 1815 * 1816 * MSIX & Interrupt Service routine 1817 * 1818 **********************************************************************/ 1819 1820 /** 1821 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue 1822 * @arg: interrupt number 1823 **/ 1824 static int 1825 ena_intr_msix_mgmnt(void *arg) 1826 { 1827 struct ena_adapter *adapter = (struct ena_adapter *)arg; 1828 1829 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); 1830 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) 1831 ena_com_aenq_intr_handler(adapter->ena_dev, arg); 1832 1833 return 1; 1834 } 1835 1836 /** 1837 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx 1838 * @arg: interrupt number 1839 **/ 1840 static int 1841 ena_handle_msix(void *arg) 1842 { 1843 struct ena_que *queue = arg; 1844 struct ena_ring *rx_ring = queue->rx_ring; 1845 1846 ENA_RING_MTX_LOCK(rx_ring); 1847 if (unlikely(rx_ring->stopping)) { 1848 ENA_RING_MTX_UNLOCK(rx_ring); 1849 return 0; 1850 } 1851 1852 if (atomic_cas_uint(&rx_ring->task_pending, 0, 1) == 0) 1853 workqueue_enqueue(rx_ring->cleanup_tq, &rx_ring->cleanup_task, 1854 curcpu()); 1855 1856 ENA_RING_MTX_UNLOCK(rx_ring); 1857 return 1; 1858 } 1859 1860 static void 1861 ena_cleanup(struct work *wk, void *arg) 1862 { 1863 struct ena_que *que = arg; 1864 struct ena_adapter *adapter = que->adapter; 1865 struct ifnet *ifp = adapter->ifp; 1866 struct ena_ring *tx_ring = que->tx_ring; 1867 struct ena_ring *rx_ring = que->rx_ring; 1868 struct ena_com_io_cq* io_cq; 1869 struct ena_eth_io_intr_reg intr_reg; 1870 int qid, ena_qid; 1871 int txc, rxc, i; 1872 1873 atomic_swap_uint(&rx_ring->task_pending, 0); 1874 1875 if (unlikely((if_getdrvflags(ifp) & IFF_RUNNING) == 0)) 1876 return; 1877 1878 ena_trace(ENA_DBG, "MSI-X TX/RX routine"); 1879 1880 qid = que->id; 1881 ena_qid = ENA_IO_TXQ_IDX(qid); 1882 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 1883 1884 for (i = 0; i < CLEAN_BUDGET; ++i) { 1885 /* 1886 * If lock cannot be acquired, then deferred cleanup task was 1887 * being executed and rx ring is being cleaned up in 1888 * another thread. 1889 */ 1890 ENA_RING_MTX_LOCK(rx_ring); 1891 if (rx_ring->stopping) { 1892 ENA_RING_MTX_UNLOCK(rx_ring); 1893 return; 1894 } 1895 ENA_RING_MTX_UNLOCK(rx_ring); 1896 rxc = ena_rx_cleanup(rx_ring); 1897 1898 /* Protection from calling ena_tx_cleanup from ena_start_xmit */ 1899 ENA_RING_MTX_LOCK(tx_ring); 1900 if (tx_ring->stopping) { 1901 ENA_RING_MTX_UNLOCK(tx_ring); 1902 return; 1903 } 1904 txc = ena_tx_cleanup(tx_ring); 1905 ENA_RING_MTX_UNLOCK(tx_ring); 1906 1907 if (unlikely((if_getdrvflags(ifp) & IFF_RUNNING) == 0)) 1908 return; 1909 1910 if ((txc != TX_BUDGET) && (rxc != RX_BUDGET)) 1911 break; 1912 } 1913 1914 /* Signal that work is done and unmask interrupt */ 1915 ena_com_update_intr_reg(&intr_reg, 1916 RX_IRQ_INTERVAL, 1917 TX_IRQ_INTERVAL, 1918 true); 1919 ena_com_unmask_intr(io_cq, &intr_reg); 1920 } 1921 1922 static int 1923 ena_enable_msix(struct ena_adapter *adapter) 1924 { 1925 int msix_req; 1926 int counts[PCI_INTR_TYPE_SIZE]; 1927 int max_type; 1928 1929 /* Reserved the max msix vectors we might need */ 1930 msix_req = ENA_MAX_MSIX_VEC(adapter->num_queues); 1931 1932 counts[PCI_INTR_TYPE_INTX] = 0; 1933 counts[PCI_INTR_TYPE_MSI] = 0; 1934 counts[PCI_INTR_TYPE_MSIX] = msix_req; 1935 max_type = PCI_INTR_TYPE_MSIX; 1936 1937 if (pci_intr_alloc(&adapter->sc_pa, &adapter->sc_intrs, counts, 1938 max_type) != 0) { 1939 aprint_error_dev(adapter->pdev, 1940 "failed to allocate interrupt\n"); 1941 return ENOSPC; 1942 } 1943 1944 adapter->sc_nintrs = counts[PCI_INTR_TYPE_MSIX]; 1945 1946 if (counts[PCI_INTR_TYPE_MSIX] != msix_req) { 1947 device_printf(adapter->pdev, 1948 "Enable only %d MSI-x (out of %d), reduce " 1949 "the number of queues\n", adapter->sc_nintrs, msix_req); 1950 adapter->num_queues = adapter->sc_nintrs - ENA_ADMIN_MSIX_VEC; 1951 } 1952 1953 return 0; 1954 } 1955 1956 #if 0 1957 static void 1958 ena_setup_io_intr(struct ena_adapter *adapter) 1959 { 1960 static int last_bind_cpu = -1; 1961 int irq_idx; 1962 1963 for (int i = 0; i < adapter->num_queues; i++) { 1964 irq_idx = ENA_IO_IRQ_IDX(i); 1965 1966 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, 1967 "%s-TxRx-%d", device_xname(adapter->pdev), i); 1968 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; 1969 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; 1970 adapter->irq_tbl[irq_idx].vector = 1971 adapter->msix_entries[irq_idx].vector; 1972 ena_trace(ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n", 1973 adapter->msix_entries[irq_idx].vector); 1974 #ifdef RSS 1975 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = 1976 rss_getcpu(i % rss_getnumbuckets()); 1977 #else 1978 /* 1979 * We still want to bind rings to the corresponding cpu 1980 * using something similar to the RSS round-robin technique. 1981 */ 1982 if (unlikely(last_bind_cpu < 0)) 1983 last_bind_cpu = CPU_FIRST(); 1984 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = 1985 last_bind_cpu; 1986 last_bind_cpu = CPU_NEXT(last_bind_cpu); 1987 #endif 1988 } 1989 } 1990 #endif 1991 1992 static int 1993 ena_request_mgmnt_irq(struct ena_adapter *adapter) 1994 { 1995 const char *intrstr; 1996 char intrbuf[PCI_INTRSTR_LEN]; 1997 char intr_xname[INTRDEVNAMEBUF]; 1998 pci_chipset_tag_t pc = adapter->sc_pa.pa_pc; 1999 const int irq_slot = ENA_MGMNT_IRQ_IDX; 2000 2001 KASSERT(adapter->sc_intrs != NULL); 2002 KASSERT(adapter->sc_ihs[irq_slot] == NULL); 2003 2004 pci_intr_setattr(pc, &adapter->sc_intrs[irq_slot], 2005 PCI_INTR_MPSAFE, true); 2006 2007 snprintf(intr_xname, sizeof(intr_xname), "%s mgmnt", 2008 device_xname(adapter->pdev)); 2009 intrstr = pci_intr_string(pc, adapter->sc_intrs[irq_slot], 2010 intrbuf, sizeof(intrbuf)); 2011 2012 adapter->sc_ihs[irq_slot] = pci_intr_establish_xname( 2013 pc, adapter->sc_intrs[irq_slot], 2014 IPL_NET, ena_intr_msix_mgmnt, adapter, intr_xname); 2015 2016 if (adapter->sc_ihs[irq_slot] == NULL) { 2017 device_printf(adapter->pdev, "failed to register " 2018 "interrupt handler for MGMNT irq %s\n", 2019 intrstr); 2020 return ENOMEM; 2021 } 2022 2023 aprint_normal_dev(adapter->pdev, 2024 "for MGMNT interrupting at %s\n", intrstr); 2025 2026 return 0; 2027 } 2028 2029 static int 2030 ena_request_io_irq(struct ena_adapter *adapter) 2031 { 2032 const char *intrstr; 2033 char intrbuf[PCI_INTRSTR_LEN]; 2034 char intr_xname[INTRDEVNAMEBUF]; 2035 pci_chipset_tag_t pc = adapter->sc_pa.pa_pc; 2036 const int irq_off = ENA_IO_IRQ_FIRST_IDX; 2037 void *vih; 2038 kcpuset_t *affinity; 2039 int i; 2040 2041 KASSERT(adapter->sc_intrs != NULL); 2042 2043 kcpuset_create(&affinity, false); 2044 2045 for (i = 0; i < adapter->num_queues; i++) { 2046 int irq_slot = i + irq_off; 2047 int affinity_to = (irq_slot) % ncpu; 2048 2049 KASSERT((void *)adapter->sc_intrs[irq_slot] != NULL); 2050 KASSERT(adapter->sc_ihs[irq_slot] == NULL); 2051 2052 pci_intr_setattr(pc, &adapter->sc_intrs[irq_slot], 2053 PCI_INTR_MPSAFE, true); 2054 2055 snprintf(intr_xname, sizeof(intr_xname), "%s ioq%d", 2056 device_xname(adapter->pdev), i); 2057 intrstr = pci_intr_string(pc, adapter->sc_intrs[irq_slot], 2058 intrbuf, sizeof(intrbuf)); 2059 2060 vih = pci_intr_establish_xname(adapter->sc_pa.pa_pc, 2061 adapter->sc_intrs[irq_slot], IPL_NET, 2062 ena_handle_msix, &adapter->que[i], intr_xname); 2063 2064 if (vih == NULL) { 2065 device_printf(adapter->pdev, "failed to register " 2066 "interrupt handler for IO queue %d irq %s\n", 2067 i, intrstr); 2068 goto err; 2069 } 2070 2071 kcpuset_zero(affinity); 2072 /* Round-robin affinity */ 2073 kcpuset_set(affinity, affinity_to); 2074 int error = interrupt_distribute(vih, affinity, NULL); 2075 if (error == 0) { 2076 aprint_normal_dev(adapter->pdev, 2077 "for IO queue %d interrupting at %s" 2078 " affinity to %u\n", i, intrstr, affinity_to); 2079 } else { 2080 aprint_normal_dev(adapter->pdev, 2081 "for IO queue %d interrupting at %s\n", i, intrstr); 2082 } 2083 2084 adapter->sc_ihs[irq_slot] = vih; 2085 2086 #ifdef RSS 2087 ena_trace(ENA_INFO, "queue %d - RSS bucket %d\n", 2088 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu); 2089 #else 2090 ena_trace(ENA_INFO, "queue %d - cpu %d\n", 2091 i - ENA_IO_IRQ_FIRST_IDX, affinity_to); 2092 #endif 2093 } 2094 2095 kcpuset_destroy(affinity); 2096 return 0; 2097 2098 err: 2099 kcpuset_destroy(affinity); 2100 2101 for (i--; i >= 0; i--) { 2102 int irq_slot __diagused = i + irq_off; 2103 KASSERT(adapter->sc_ihs[irq_slot] != NULL); 2104 pci_intr_disestablish(adapter->sc_pa.pa_pc, adapter->sc_ihs[irq_slot]); 2105 adapter->sc_ihs[irq_slot] = NULL; 2106 } 2107 2108 return ENOSPC; 2109 } 2110 2111 static void 2112 ena_free_mgmnt_irq(struct ena_adapter *adapter) 2113 { 2114 const int irq_slot = ENA_MGMNT_IRQ_IDX; 2115 2116 if (adapter->sc_ihs[irq_slot]) { 2117 pci_intr_disestablish(adapter->sc_pa.pa_pc, 2118 adapter->sc_ihs[irq_slot]); 2119 adapter->sc_ihs[irq_slot] = NULL; 2120 } 2121 } 2122 2123 static void 2124 ena_free_io_irq(struct ena_adapter *adapter) 2125 { 2126 const int irq_off = ENA_IO_IRQ_FIRST_IDX; 2127 2128 for (int i = 0; i < adapter->num_queues; i++) { 2129 int irq_slot = i + irq_off; 2130 2131 if (adapter->sc_ihs[irq_slot]) { 2132 pci_intr_disestablish(adapter->sc_pa.pa_pc, 2133 adapter->sc_ihs[irq_slot]); 2134 adapter->sc_ihs[irq_slot] = NULL; 2135 } 2136 } 2137 } 2138 2139 static void 2140 ena_free_irqs(struct ena_adapter* adapter) 2141 { 2142 2143 ena_free_io_irq(adapter); 2144 ena_free_mgmnt_irq(adapter); 2145 ena_disable_msix(adapter); 2146 } 2147 2148 static void 2149 ena_disable_msix(struct ena_adapter *adapter) 2150 { 2151 pci_intr_release(adapter->sc_pa.pa_pc, adapter->sc_intrs, 2152 adapter->sc_nintrs); 2153 } 2154 2155 static void 2156 ena_unmask_all_io_irqs(struct ena_adapter *adapter) 2157 { 2158 struct ena_com_io_cq* io_cq; 2159 struct ena_eth_io_intr_reg intr_reg; 2160 uint16_t ena_qid; 2161 int i; 2162 2163 /* Unmask interrupts for all queues */ 2164 for (i = 0; i < adapter->num_queues; i++) { 2165 ena_qid = ENA_IO_TXQ_IDX(i); 2166 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 2167 ena_com_update_intr_reg(&intr_reg, 0, 0, true); 2168 ena_com_unmask_intr(io_cq, &intr_reg); 2169 } 2170 } 2171 2172 /* Configure the Rx forwarding */ 2173 static int 2174 ena_rss_configure(struct ena_adapter *adapter) 2175 { 2176 struct ena_com_dev *ena_dev = adapter->ena_dev; 2177 int rc; 2178 2179 /* Set indirect table */ 2180 rc = ena_com_indirect_table_set(ena_dev); 2181 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 2182 return (rc); 2183 2184 /* Configure hash function (if supported) */ 2185 rc = ena_com_set_hash_function(ena_dev); 2186 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 2187 return (rc); 2188 2189 /* Configure hash inputs (if supported) */ 2190 rc = ena_com_set_hash_ctrl(ena_dev); 2191 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 2192 return (rc); 2193 2194 return (0); 2195 } 2196 2197 static int 2198 ena_up_complete(struct ena_adapter *adapter) 2199 { 2200 int rc; 2201 2202 if (likely(adapter->rss_support)) { 2203 rc = ena_rss_configure(adapter); 2204 if (rc != 0) 2205 return (rc); 2206 } 2207 2208 rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu); 2209 if (unlikely(rc != 0)) 2210 return (rc); 2211 2212 ena_refill_all_rx_bufs(adapter); 2213 ena_reset_counters((struct evcnt *)&adapter->hw_stats, 2214 sizeof(adapter->hw_stats), 2215 offsetof(struct ena_hw_stats, rx_packets)); 2216 2217 return (0); 2218 } 2219 2220 static int 2221 ena_up(struct ena_adapter *adapter) 2222 { 2223 int rc = 0; 2224 2225 KASSERT(ENA_CORE_MTX_OWNED(adapter)); 2226 2227 #if 0 2228 if (unlikely(device_is_attached(adapter->pdev) == 0)) { 2229 device_printf(adapter->pdev, "device is not attached!\n"); 2230 return (ENXIO); 2231 } 2232 #endif 2233 2234 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) { 2235 device_printf(adapter->pdev, "device is not running!\n"); 2236 return (ENXIO); 2237 } 2238 2239 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2240 device_printf(adapter->pdev, "device is going UP\n"); 2241 2242 /* allocate transmit descriptors */ 2243 rc = ena_setup_all_tx_resources(adapter); 2244 if (unlikely(rc != 0)) { 2245 ena_trace(ENA_ALERT, "err_setup_tx"); 2246 goto err_setup_tx; 2247 } 2248 2249 /* allocate receive descriptors */ 2250 rc = ena_setup_all_rx_resources(adapter); 2251 if (unlikely(rc != 0)) { 2252 ena_trace(ENA_ALERT, "err_setup_rx"); 2253 goto err_setup_rx; 2254 } 2255 2256 /* create IO queues for Rx & Tx */ 2257 rc = ena_create_io_queues(adapter); 2258 if (unlikely(rc != 0)) { 2259 ena_trace(ENA_ALERT, 2260 "create IO queues failed"); 2261 goto err_io_que; 2262 } 2263 2264 /* setup interrupts for IO queues */ 2265 rc = ena_request_io_irq(adapter); 2266 if (unlikely(rc != 0)) { 2267 ena_trace(ENA_ALERT, "err_req_irq"); 2268 goto err_req_irq; 2269 } 2270 2271 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))) 2272 if_link_state_change(adapter->ifp, LINK_STATE_UP); 2273 2274 rc = ena_up_complete(adapter); 2275 if (unlikely(rc != 0)) 2276 goto err_up_complete; 2277 2278 counter_u64_add(adapter->dev_stats.interface_up, 1); 2279 2280 ena_update_hwassist(adapter); 2281 2282 if_setdrvflagbits(adapter->ifp, IFF_RUNNING, 2283 IFF_OACTIVE); 2284 ena_set_stopping_flag(adapter, false); 2285 2286 callout_schedule(&adapter->timer_service, hz); 2287 2288 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2289 2290 ena_unmask_all_io_irqs(adapter); 2291 } 2292 2293 return (0); 2294 2295 err_up_complete: 2296 ena_destroy_all_io_queues(adapter); 2297 err_io_que: 2298 ena_free_all_rx_resources(adapter); 2299 err_setup_rx: 2300 ena_free_all_tx_resources(adapter); 2301 err_setup_tx: 2302 ena_free_io_irq(adapter); 2303 err_req_irq: 2304 return (rc); 2305 } 2306 2307 #if 0 2308 static uint64_t 2309 ena_get_counter(struct ifnet *ifp, ift_counter cnt) 2310 { 2311 struct ena_adapter *adapter; 2312 struct ena_hw_stats *stats; 2313 2314 adapter = if_getsoftc(ifp); 2315 stats = &adapter->hw_stats; 2316 2317 switch (cnt) { 2318 case IFCOUNTER_IPACKETS: 2319 return (counter_u64_fetch(stats->rx_packets)); 2320 case IFCOUNTER_OPACKETS: 2321 return (counter_u64_fetch(stats->tx_packets)); 2322 case IFCOUNTER_IBYTES: 2323 return (counter_u64_fetch(stats->rx_bytes)); 2324 case IFCOUNTER_OBYTES: 2325 return (counter_u64_fetch(stats->tx_bytes)); 2326 case IFCOUNTER_IQDROPS: 2327 return (counter_u64_fetch(stats->rx_drops)); 2328 default: 2329 return (if_get_counter_default(ifp, cnt)); 2330 } 2331 } 2332 #endif 2333 2334 static int 2335 ena_media_change(struct ifnet *ifp) 2336 { 2337 /* Media Change is not supported by firmware */ 2338 return (0); 2339 } 2340 2341 static void 2342 ena_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2343 { 2344 struct ena_adapter *adapter = if_getsoftc(ifp); 2345 ena_trace(ENA_DBG, "enter"); 2346 2347 KASSERT(ENA_CORE_MTX_OWNED(adapter)); 2348 2349 ifmr->ifm_status = IFM_AVALID; 2350 ifmr->ifm_active = IFM_ETHER; 2351 2352 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) { 2353 ena_trace(ENA_INFO, "Link is down"); 2354 return; 2355 } 2356 2357 ifmr->ifm_status |= IFM_ACTIVE; 2358 } 2359 2360 static int 2361 ena_init(struct ifnet *ifp) 2362 { 2363 struct ena_adapter *adapter = if_getsoftc(ifp); 2364 2365 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2366 ENA_CORE_MTX_LOCK(adapter); 2367 ena_up(adapter); 2368 ENA_CORE_MTX_UNLOCK(adapter); 2369 } 2370 2371 return 0; 2372 } 2373 2374 static void 2375 ena_stop(struct ifnet *ifp, int disable){ 2376 struct ena_adapter *adapter; 2377 2378 adapter = ifp->if_softc; 2379 2380 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2381 ENA_CORE_MTX_LOCK(adapter); 2382 ena_down(adapter); 2383 ENA_CORE_MTX_UNLOCK(adapter); 2384 } 2385 } 2386 2387 2388 static int 2389 ena_ioctl(struct ifnet *ifp, u_long command, void *data) 2390 { 2391 struct ena_adapter *adapter; 2392 struct ifreq *ifr; 2393 int rc; 2394 2395 adapter = ifp->if_softc; 2396 ifr = (struct ifreq *)data; 2397 2398 /* 2399 * Acquiring lock to prevent from running up and down routines parallel. 2400 */ 2401 rc = 0; 2402 switch (command) { 2403 case SIOCSIFMTU: 2404 if (ifp->if_mtu == ifr->ifr_mtu) 2405 break; 2406 ENA_CORE_MTX_LOCK(adapter); 2407 ena_down(adapter); 2408 2409 ena_change_mtu(ifp, ifr->ifr_mtu); 2410 2411 rc = ena_up(adapter); 2412 ENA_CORE_MTX_UNLOCK(adapter); 2413 break; 2414 2415 case SIOCADDMULTI: 2416 case SIOCDELMULTI: 2417 break; 2418 2419 case SIOCSIFCAP: 2420 { 2421 struct ifcapreq *ifcr = data; 2422 int reinit = 0; 2423 2424 if (ifcr->ifcr_capenable != ifp->if_capenable) { 2425 ifp->if_capenable = ifcr->ifcr_capenable; 2426 reinit = 1; 2427 } 2428 2429 if ((reinit != 0) && 2430 ((if_getdrvflags(ifp) & IFF_RUNNING) != 0)) { 2431 ENA_CORE_MTX_LOCK(adapter); 2432 ena_down(adapter); 2433 rc = ena_up(adapter); 2434 ENA_CORE_MTX_UNLOCK(adapter); 2435 } 2436 } 2437 2438 break; 2439 default: 2440 rc = ether_ioctl(ifp, command, data); 2441 break; 2442 } 2443 2444 return (rc); 2445 } 2446 2447 static int 2448 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat) 2449 { 2450 int caps = 0; 2451 2452 if ((feat->offload.tx & 2453 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2454 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK | 2455 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0) 2456 caps |= IFCAP_CSUM_IPv4_Tx; 2457 2458 if ((feat->offload.tx & 2459 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK | 2460 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0) 2461 caps |= IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_UDPv6_Tx; 2462 2463 if ((feat->offload.tx & 2464 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0) 2465 caps |= IFCAP_TSOv4; 2466 2467 if ((feat->offload.tx & 2468 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0) 2469 caps |= IFCAP_TSOv6; 2470 2471 if ((feat->offload.rx_supported & 2472 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK | 2473 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0) 2474 caps |= IFCAP_CSUM_IPv4_Rx; 2475 2476 if ((feat->offload.rx_supported & 2477 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0) 2478 caps |= IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 2479 2480 #ifdef LRO 2481 caps |= IFCAP_LRO; 2482 #endif 2483 2484 return (caps); 2485 } 2486 2487 static void 2488 ena_update_host_info(struct ena_admin_host_info *host_info, struct ifnet *ifp) 2489 { 2490 2491 host_info->supported_network_features[0] = 2492 (uint32_t)if_getcapabilities(ifp); 2493 } 2494 2495 static void 2496 ena_update_hwassist(struct ena_adapter *adapter) 2497 { 2498 struct ifnet *ifp = adapter->ifp; 2499 uint32_t feat = adapter->tx_offload_cap; 2500 int cap = if_getcapenable(ifp); 2501 int flags = 0; 2502 2503 if_clearhwassist(ifp); 2504 2505 if ((cap & (IFCAP_CSUM_IPv4_Tx|IFCAP_CSUM_TCPv4_Tx|IFCAP_CSUM_UDPv4_Tx)) 2506 != 0) { 2507 if ((feat & 2508 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0) 2509 flags |= M_CSUM_IPv4; 2510 if ((feat & 2511 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2512 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0) 2513 flags |= M_CSUM_TCPv4 | M_CSUM_UDPv4; 2514 } 2515 2516 if ((cap & IFCAP_CSUM_TCPv6_Tx) != 0) 2517 flags |= M_CSUM_TCPv6; 2518 2519 if ((cap & IFCAP_CSUM_UDPv6_Tx) != 0) 2520 flags |= M_CSUM_UDPv6; 2521 2522 if ((cap & IFCAP_TSOv4) != 0) 2523 flags |= M_CSUM_TSOv4; 2524 2525 if ((cap & IFCAP_TSOv6) != 0) 2526 flags |= M_CSUM_TSOv6; 2527 2528 if_sethwassistbits(ifp, flags, 0); 2529 } 2530 2531 static int 2532 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter, 2533 struct ena_com_dev_get_features_ctx *feat) 2534 { 2535 struct ifnet *ifp; 2536 int caps = 0; 2537 2538 ifp = adapter->ifp = &adapter->sc_ec.ec_if; 2539 if (unlikely(ifp == NULL)) { 2540 ena_trace(ENA_ALERT, "can not allocate ifnet structure\n"); 2541 return (ENXIO); 2542 } 2543 if_initialize(ifp); 2544 if_initname(ifp, "ena", device_unit(pdev)); 2545 if_setdev(ifp, pdev); 2546 if_setsoftc(ifp, adapter); 2547 2548 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2549 ifp->if_extflags = IFEF_MPSAFE; 2550 if_setinitfn(ifp, ena_init); 2551 ifp->if_stop = ena_stop; 2552 if_settransmitfn(ifp, ena_mq_start); 2553 #if 0 2554 if_setqflushfn(ifp, ena_qflush); 2555 #endif 2556 if_setioctlfn(ifp, ena_ioctl); 2557 #if 0 2558 if_setgetcounterfn(ifp, ena_get_counter); 2559 #endif 2560 2561 if_setsendqlen(ifp, adapter->tx_ring_size); 2562 if_setsendqready(ifp); 2563 if_setmtu(ifp, ETHERMTU); 2564 if_setbaudrate(ifp, 0); 2565 /* Zeroize capabilities... */ 2566 if_setcapabilities(ifp, 0); 2567 if_setcapenable(ifp, 0); 2568 /* check hardware support */ 2569 caps = ena_get_dev_offloads(feat); 2570 /* ... and set them */ 2571 if_setcapabilitiesbit(ifp, caps, 0); 2572 adapter->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU; 2573 2574 #if 0 2575 /* TSO parameters */ 2576 /* XXX no limits on NetBSD, guarded by virtue of dmamap load failing */ 2577 ifp->if_hw_tsomax = ENA_TSO_MAXSIZE - 2578 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2579 ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1; 2580 ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE; 2581 #endif 2582 2583 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2584 if_setcapenable(ifp, if_getcapabilities(ifp)); 2585 2586 /* 2587 * Specify the media types supported by this adapter and register 2588 * callbacks to update media and link information 2589 */ 2590 adapter->sc_ec.ec_ifmedia = &adapter->media; 2591 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, 2592 ena_media_change, ena_media_status, &adapter->global_mtx); 2593 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2594 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2595 2596 ifp->if_percpuq = if_percpuq_create(ifp); 2597 if_deferred_start_init(ifp, NULL); 2598 ether_ifattach(ifp, adapter->mac_addr); 2599 if_register(ifp); 2600 2601 return (0); 2602 } 2603 2604 static void 2605 ena_set_stopping_flag(struct ena_adapter *adapter, bool value) 2606 { 2607 struct ena_ring *ring; 2608 int i; 2609 2610 for (i = 0; i < adapter->num_queues; i++) { 2611 /* TX */ 2612 ring = adapter->que[i].tx_ring; 2613 ENA_RING_MTX_LOCK(ring); 2614 ring->stopping = value; 2615 ENA_RING_MTX_UNLOCK(ring); 2616 2617 /* RX */ 2618 ring = adapter->que[i].rx_ring; 2619 ENA_RING_MTX_LOCK(ring); 2620 ring->stopping = value; 2621 ENA_RING_MTX_UNLOCK(ring); 2622 } 2623 } 2624 2625 static void 2626 ena_down(struct ena_adapter *adapter) 2627 { 2628 int rc, i; 2629 2630 KASSERT(ENA_CORE_MTX_OWNED(adapter)); 2631 2632 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2633 device_printf(adapter->pdev, "device is going DOWN\n"); 2634 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2635 if_setdrvflagbits(adapter->ifp, IFF_OACTIVE, 2636 IFF_RUNNING); 2637 2638 ena_set_stopping_flag(adapter, true); 2639 2640 callout_halt(&adapter->timer_service, NULL); 2641 for (i = 0; i < adapter->num_queues; i++) { 2642 struct ena_ring *rx_ring = adapter->que[i].rx_ring; 2643 workqueue_wait(rx_ring->cleanup_tq, 2644 &rx_ring->cleanup_task); 2645 } 2646 2647 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) { 2648 rc = ena_com_dev_reset(adapter->ena_dev, 2649 adapter->reset_reason); 2650 if (unlikely(rc != 0)) 2651 device_printf(adapter->pdev, 2652 "Device reset failed\n"); 2653 } 2654 2655 ena_destroy_all_io_queues(adapter); 2656 2657 ena_free_all_tx_bufs(adapter); 2658 ena_free_all_rx_bufs(adapter); 2659 ena_free_all_tx_resources(adapter); 2660 ena_free_all_rx_resources(adapter); 2661 2662 ena_free_io_irq(adapter); 2663 2664 counter_u64_add(adapter->dev_stats.interface_down, 1); 2665 } 2666 } 2667 2668 static void 2669 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf) 2670 { 2671 struct ena_com_tx_meta *ena_meta; 2672 struct ether_vlan_header *eh; 2673 u32 mss; 2674 bool offload; 2675 uint16_t etype; 2676 int ehdrlen; 2677 struct ip *ip; 2678 int iphlen; 2679 struct tcphdr *th; 2680 2681 offload = false; 2682 ena_meta = &ena_tx_ctx->ena_meta; 2683 2684 #if 0 2685 u32 mss = mbuf->m_pkthdr.tso_segsz; 2686 2687 if (mss != 0) 2688 offload = true; 2689 #else 2690 mss = mbuf->m_pkthdr.len; /* XXX don't have tso_segsz */ 2691 #endif 2692 2693 if ((mbuf->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0) 2694 offload = true; 2695 2696 if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0) 2697 offload = true; 2698 2699 if (!offload) { 2700 ena_tx_ctx->meta_valid = 0; 2701 return; 2702 } 2703 2704 /* Determine where frame payload starts. */ 2705 eh = mtod(mbuf, struct ether_vlan_header *); 2706 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2707 etype = ntohs(eh->evl_proto); 2708 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2709 } else { 2710 etype = htons(eh->evl_encap_proto); 2711 ehdrlen = ETHER_HDR_LEN; 2712 } 2713 2714 ip = (struct ip *)(mbuf->m_data + ehdrlen); 2715 iphlen = ip->ip_hl << 2; 2716 th = (struct tcphdr *)((vaddr_t)ip + iphlen); 2717 2718 if ((mbuf->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) { 2719 ena_tx_ctx->l3_csum_enable = 1; 2720 } 2721 if ((mbuf->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0) { 2722 ena_tx_ctx->tso_enable = 1; 2723 ena_meta->l4_hdr_len = (th->th_off); 2724 } 2725 2726 switch (etype) { 2727 case ETHERTYPE_IP: 2728 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 2729 if ((ip->ip_off & htons(IP_DF)) != 0) 2730 ena_tx_ctx->df = 1; 2731 break; 2732 case ETHERTYPE_IPV6: 2733 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 2734 2735 default: 2736 break; 2737 } 2738 2739 if (ip->ip_p == IPPROTO_TCP) { 2740 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 2741 if ((mbuf->m_pkthdr.csum_flags & 2742 (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0) 2743 ena_tx_ctx->l4_csum_enable = 1; 2744 else 2745 ena_tx_ctx->l4_csum_enable = 0; 2746 } else if (ip->ip_p == IPPROTO_UDP) { 2747 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 2748 if ((mbuf->m_pkthdr.csum_flags & 2749 (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0) 2750 ena_tx_ctx->l4_csum_enable = 1; 2751 else 2752 ena_tx_ctx->l4_csum_enable = 0; 2753 } else { 2754 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 2755 ena_tx_ctx->l4_csum_enable = 0; 2756 } 2757 2758 ena_meta->mss = mss; 2759 ena_meta->l3_hdr_len = iphlen; 2760 ena_meta->l3_hdr_offset = ehdrlen; 2761 ena_tx_ctx->meta_valid = 1; 2762 } 2763 2764 static int 2765 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 2766 { 2767 struct ena_adapter *adapter; 2768 struct mbuf *collapsed_mbuf; 2769 int num_frags; 2770 2771 adapter = tx_ring->adapter; 2772 num_frags = ena_mbuf_count(*mbuf); 2773 2774 /* One segment must be reserved for configuration descriptor. */ 2775 if (num_frags < adapter->max_tx_sgl_size) 2776 return (0); 2777 counter_u64_add(tx_ring->tx_stats.collapse, 1); 2778 2779 collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT, 2780 adapter->max_tx_sgl_size - 1); 2781 if (unlikely(collapsed_mbuf == NULL)) { 2782 counter_u64_add(tx_ring->tx_stats.collapse_err, 1); 2783 return (ENOMEM); 2784 } 2785 2786 /* If mbuf was collapsed successfully, original mbuf is released. */ 2787 *mbuf = collapsed_mbuf; 2788 2789 return (0); 2790 } 2791 2792 static int 2793 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf) 2794 { 2795 struct ena_adapter *adapter; 2796 struct ena_tx_buffer *tx_info; 2797 struct ena_com_tx_ctx ena_tx_ctx; 2798 struct ena_com_dev *ena_dev; 2799 struct ena_com_buf *ena_buf; 2800 struct ena_com_io_sq* io_sq; 2801 void *push_hdr; 2802 uint16_t next_to_use; 2803 uint16_t req_id; 2804 uint16_t ena_qid; 2805 uint32_t header_len; 2806 int i, rc; 2807 int nb_hw_desc; 2808 2809 KASSERT(ENA_RING_MTX_OWNED(tx_ring)); 2810 2811 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 2812 adapter = tx_ring->que->adapter; 2813 ena_dev = adapter->ena_dev; 2814 io_sq = &ena_dev->io_sq_queues[ena_qid]; 2815 2816 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf); 2817 if (unlikely(rc != 0)) { 2818 ena_trace(ENA_WARNING, 2819 "Failed to collapse mbuf! err: %d", rc); 2820 return (rc); 2821 } 2822 2823 next_to_use = tx_ring->next_to_use; 2824 req_id = tx_ring->free_tx_ids[next_to_use]; 2825 tx_info = &tx_ring->tx_buffer_info[req_id]; 2826 2827 tx_info->mbuf = *mbuf; 2828 tx_info->num_of_bufs = 0; 2829 2830 ena_buf = tx_info->bufs; 2831 2832 ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes", (*mbuf)->m_pkthdr.len); 2833 2834 /* 2835 * header_len is just a hint for the device. Because FreeBSD is not 2836 * giving us information about packet header length and it is not 2837 * guaranteed that all packet headers will be in the 1st mbuf, setting 2838 * header_len to 0 is making the device ignore this value and resolve 2839 * header on it's own. 2840 */ 2841 header_len = 0; 2842 push_hdr = NULL; 2843 2844 rc = bus_dmamap_load_mbuf(adapter->sc_dmat, tx_info->map, 2845 *mbuf, BUS_DMA_NOWAIT); 2846 2847 if (unlikely((rc != 0) || (tx_info->map->dm_nsegs == 0))) { 2848 ena_trace(ENA_WARNING, 2849 "dmamap load failed! err: %d nsegs: %d", rc, 2850 tx_info->map->dm_nsegs); 2851 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1); 2852 tx_info->mbuf = NULL; 2853 if (rc == ENOMEM) 2854 return (ENA_COM_NO_MEM); 2855 else 2856 return (ENA_COM_INVAL); 2857 } 2858 2859 for (i = 0; i < tx_info->map->dm_nsegs; i++) { 2860 ena_buf->len = tx_info->map->dm_segs[i].ds_len; 2861 ena_buf->paddr = tx_info->map->dm_segs[i].ds_addr; 2862 ena_buf++; 2863 } 2864 tx_info->num_of_bufs = tx_info->map->dm_nsegs; 2865 2866 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 2867 ena_tx_ctx.ena_bufs = tx_info->bufs; 2868 ena_tx_ctx.push_header = push_hdr; 2869 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 2870 ena_tx_ctx.req_id = req_id; 2871 ena_tx_ctx.header_len = header_len; 2872 2873 /* Set flags and meta data */ 2874 ena_tx_csum(&ena_tx_ctx, *mbuf); 2875 /* Prepare the packet's descriptors and send them to device */ 2876 rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc); 2877 if (unlikely(rc != 0)) { 2878 device_printf(adapter->pdev, "failed to prepare tx bufs\n"); 2879 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1); 2880 goto dma_error; 2881 } 2882 2883 counter_enter(); 2884 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1); 2885 counter_u64_add_protected(tx_ring->tx_stats.bytes, 2886 (*mbuf)->m_pkthdr.len); 2887 2888 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1); 2889 counter_u64_add_protected(adapter->hw_stats.tx_bytes, 2890 (*mbuf)->m_pkthdr.len); 2891 counter_exit(); 2892 2893 tx_info->tx_descs = nb_hw_desc; 2894 getbinuptime(&tx_info->timestamp); 2895 tx_info->print_once = true; 2896 2897 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, 2898 tx_ring->ring_size); 2899 2900 bus_dmamap_sync(adapter->sc_dmat, tx_info->map, 0, 2901 tx_info->map->dm_mapsize, BUS_DMASYNC_PREWRITE); 2902 2903 return (0); 2904 2905 dma_error: 2906 tx_info->mbuf = NULL; 2907 bus_dmamap_unload(adapter->sc_dmat, tx_info->map); 2908 2909 return (rc); 2910 } 2911 2912 static void 2913 ena_start_xmit(struct ena_ring *tx_ring) 2914 { 2915 struct mbuf *mbuf; 2916 struct ena_adapter *adapter = tx_ring->adapter; 2917 struct ena_com_io_sq* io_sq; 2918 int ena_qid; 2919 int acum_pkts = 0; 2920 int ret = 0; 2921 net_stat_ref_t nsr; 2922 2923 KASSERT(ENA_RING_MTX_OWNED(tx_ring)); 2924 2925 /* ena_down() is waiting for completing */ 2926 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_RUNNING) == 0)) 2927 return; 2928 2929 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))) 2930 return; 2931 2932 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id); 2933 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid]; 2934 2935 nsr = IF_STAT_GETREF(adapter->ifp); 2936 2937 for (;;) { 2938 if (unlikely(!ena_com_sq_have_enough_space(io_sq, adapter->max_tx_sgl_size))) 2939 break; 2940 2941 if ((mbuf = pcq_get(tx_ring->br)) == NULL) 2942 break; 2943 2944 ena_trace(ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and" 2945 " header csum flags %#jx", 2946 mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags); 2947 2948 if (likely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) == 0)) { 2949 if_statinc_ref(nsr, if_opackets); 2950 if_statadd_ref(nsr, if_obytes, mbuf->m_pkthdr.len); 2951 if (ISSET(mbuf->m_flags, M_MCAST)) 2952 if_statinc_ref(nsr, if_omcasts); 2953 } else { 2954 if_statinc_ref(nsr, if_oerrors); 2955 m_freem(mbuf); 2956 break; 2957 } 2958 2959 /* ena_down is waiting for completing */ 2960 if (unlikely((if_getdrvflags(adapter->ifp) & 2961 IFF_RUNNING) == 0)) { 2962 IF_STAT_PUTREF(adapter->ifp); 2963 return; 2964 } 2965 2966 acum_pkts++; 2967 2968 /* 2969 * If there's a BPF listener, bounce a copy of this frame 2970 * to him. 2971 */ 2972 bpf_mtap(adapter->ifp, mbuf, BPF_D_OUT); 2973 2974 if (unlikely(acum_pkts == DB_THRESHOLD)) { 2975 acum_pkts = 0; 2976 wmb(); 2977 /* Trigger the dma engine */ 2978 ena_com_write_sq_doorbell(io_sq); 2979 counter_u64_add(tx_ring->tx_stats.doorbells, 1); 2980 } 2981 2982 } 2983 2984 IF_STAT_PUTREF(adapter->ifp); 2985 2986 if (likely(acum_pkts != 0)) { 2987 wmb(); 2988 /* Trigger the dma engine */ 2989 ena_com_write_sq_doorbell(io_sq); 2990 counter_u64_add(tx_ring->tx_stats.doorbells, 1); 2991 } 2992 2993 if (!ena_com_sq_have_enough_space(io_sq, ENA_TX_CLEANUP_THRESHOLD)) 2994 ena_tx_cleanup(tx_ring); 2995 } 2996 2997 static void 2998 ena_deferred_mq_start(struct work *wk, void *arg) 2999 { 3000 struct ena_ring *tx_ring = (struct ena_ring *)arg; 3001 struct ifnet *ifp = tx_ring->adapter->ifp; 3002 3003 atomic_swap_uint(&tx_ring->task_pending, 0); 3004 3005 while (pcq_peek(tx_ring->br) != NULL && 3006 (if_getdrvflags(ifp) & IFF_RUNNING) != 0) { 3007 ENA_RING_MTX_LOCK(tx_ring); 3008 if (tx_ring->stopping) { 3009 ENA_RING_MTX_UNLOCK(tx_ring); 3010 return; 3011 } 3012 ena_start_xmit(tx_ring); 3013 ENA_RING_MTX_UNLOCK(tx_ring); 3014 } 3015 } 3016 3017 static int 3018 ena_mq_start(struct ifnet *ifp, struct mbuf *m) 3019 { 3020 struct ena_adapter *adapter = ifp->if_softc; 3021 struct ena_ring *tx_ring; 3022 struct mbuf *is_drbr_empty; 3023 bool ret; 3024 uint32_t i; 3025 3026 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_RUNNING) == 0)) 3027 return (ENODEV); 3028 3029 /* Which queue to use */ 3030 /* 3031 * If everything is setup correctly, it should be the 3032 * same bucket that the current CPU we're on is. 3033 * It should improve performance. 3034 */ 3035 i = cpu_index(curcpu()) % adapter->num_queues; 3036 3037 tx_ring = &adapter->tx_ring[i]; 3038 3039 /* Check if drbr is empty before putting packet */ 3040 is_drbr_empty = pcq_peek(tx_ring->br); 3041 ret = pcq_put(tx_ring->br, m); 3042 if (unlikely(ret == false)) { 3043 m_freem(m); 3044 counter_u64_add(tx_ring->tx_stats.pcq_drops, 1); 3045 if (atomic_cas_uint(&tx_ring->task_pending, 0, 1) == 0) 3046 workqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task, 3047 curcpu()); 3048 return (ENOBUFS); 3049 } 3050 3051 if ((is_drbr_empty != NULL) && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) { 3052 if (!tx_ring->stopping) 3053 ena_start_xmit(tx_ring); 3054 ENA_RING_MTX_UNLOCK(tx_ring); 3055 } else { 3056 if (atomic_cas_uint(&tx_ring->task_pending, 0, 1) == 0) 3057 workqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task, 3058 curcpu()); 3059 } 3060 3061 return (0); 3062 } 3063 3064 #if 0 3065 static void 3066 ena_qflush(struct ifnet *ifp) 3067 { 3068 struct ena_adapter *adapter = ifp->if_softc; 3069 struct ena_ring *tx_ring = adapter->tx_ring; 3070 int i; 3071 3072 for(i = 0; i < adapter->num_queues; ++i, ++tx_ring) 3073 if (!drbr_empty(ifp, tx_ring->br)) { 3074 ENA_RING_MTX_LOCK(tx_ring); 3075 drbr_flush(ifp, tx_ring->br); 3076 ENA_RING_MTX_UNLOCK(tx_ring); 3077 } 3078 3079 if_qflush(ifp); 3080 } 3081 #endif 3082 3083 static int 3084 ena_calc_io_queue_num(struct pci_attach_args *pa, 3085 struct ena_adapter *adapter, 3086 struct ena_com_dev_get_features_ctx *get_feat_ctx) 3087 { 3088 int io_sq_num, io_cq_num, io_queue_num; 3089 3090 io_sq_num = get_feat_ctx->max_queues.max_sq_num; 3091 io_cq_num = get_feat_ctx->max_queues.max_cq_num; 3092 3093 io_queue_num = min_t(int, mp_ncpus, ENA_MAX_NUM_IO_QUEUES); 3094 io_queue_num = min_t(int, io_queue_num, io_sq_num); 3095 io_queue_num = min_t(int, io_queue_num, io_cq_num); 3096 /* 1 IRQ for mgmnt and 1 IRQ for each TX/RX pair */ 3097 io_queue_num = min_t(int, io_queue_num, 3098 pci_msix_count(pa->pa_pc, pa->pa_tag) - 1); 3099 3100 return (io_queue_num); 3101 } 3102 3103 static int 3104 ena_calc_queue_size(struct ena_adapter *adapter, uint16_t *max_tx_sgl_size, 3105 uint16_t *max_rx_sgl_size, struct ena_com_dev_get_features_ctx *feat) 3106 { 3107 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 3108 uint32_t v; 3109 uint32_t q; 3110 3111 queue_size = min_t(uint32_t, queue_size, 3112 feat->max_queues.max_cq_depth); 3113 queue_size = min_t(uint32_t, queue_size, 3114 feat->max_queues.max_sq_depth); 3115 3116 /* round down to the nearest power of 2 */ 3117 v = queue_size; 3118 while (v != 0) { 3119 if (powerof2(queue_size) != 0) 3120 break; 3121 v /= 2; 3122 q = rounddown2(queue_size, v); 3123 if (q != 0) { 3124 queue_size = q; 3125 break; 3126 } 3127 } 3128 3129 if (unlikely(queue_size == 0)) { 3130 device_printf(adapter->pdev, "Invalid queue size\n"); 3131 return (ENA_COM_FAULT); 3132 } 3133 3134 *max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 3135 feat->max_queues.max_packet_tx_descs); 3136 *max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 3137 feat->max_queues.max_packet_rx_descs); 3138 3139 return (queue_size); 3140 } 3141 3142 static void 3143 ena_rss_init_default(device_t self) 3144 { 3145 struct ena_adapter *adapter = device_private(self); 3146 struct ena_com_dev *ena_dev = adapter->ena_dev; 3147 device_t dev = adapter->pdev; 3148 int qid, rc, i; 3149 3150 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 3151 if (unlikely(rc != 0)) { 3152 device_printf(dev, "Cannot init indirect table\n"); 3153 return; 3154 } 3155 3156 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 3157 qid = i % adapter->num_queues; 3158 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 3159 ENA_IO_RXQ_IDX(qid)); 3160 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 3161 device_printf(dev, "Cannot fill indirect table\n"); 3162 goto err_rss_destroy; 3163 } 3164 } 3165 3166 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 3167 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 3168 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 3169 device_printf(dev, "Cannot fill hash function\n"); 3170 goto err_rss_destroy; 3171 } 3172 3173 rc = ena_com_set_default_hash_ctrl(ena_dev); 3174 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 3175 device_printf(dev, "Cannot fill hash control\n"); 3176 goto err_rss_destroy; 3177 } 3178 3179 adapter->rss_support = true; 3180 return; 3181 3182 err_rss_destroy: 3183 ena_com_rss_destroy(ena_dev); 3184 adapter->rss_support = false; 3185 return; 3186 } 3187 3188 static void 3189 ena_config_host_info(struct ena_com_dev *ena_dev) 3190 { 3191 struct ena_admin_host_info *host_info; 3192 int rc; 3193 3194 /* Allocate only the host info */ 3195 rc = ena_com_allocate_host_info(ena_dev); 3196 if (unlikely(rc != 0)) { 3197 ena_trace(ENA_ALERT, "Cannot allocate host info\n"); 3198 return; 3199 } 3200 3201 host_info = ena_dev->host_attr.host_info; 3202 3203 host_info->os_type = ENA_ADMIN_OS_FREEBSD; 3204 host_info->kernel_ver = osreldate; 3205 3206 snprintf(host_info->kernel_ver_str, sizeof(host_info->kernel_ver_str), 3207 "%d", osreldate); 3208 host_info->os_dist = 0; 3209 strncpy(host_info->os_dist_str, osrelease, 3210 sizeof(host_info->os_dist_str) - 1); 3211 3212 host_info->driver_version = 3213 (DRV_MODULE_VER_MAJOR) | 3214 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 3215 (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 3216 3217 rc = ena_com_set_host_attributes(ena_dev); 3218 if (unlikely(rc != 0)) { 3219 if (rc == EOPNOTSUPP) 3220 ena_trace(ENA_WARNING, "Cannot set host attributes\n"); 3221 else 3222 ena_trace(ENA_ALERT, "Cannot set host attributes\n"); 3223 3224 goto err; 3225 } 3226 3227 return; 3228 3229 err: 3230 ena_com_delete_host_info(ena_dev); 3231 } 3232 3233 static int 3234 ena_device_init(struct ena_adapter *adapter, device_t pdev, 3235 struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active) 3236 { 3237 struct ena_com_dev* ena_dev = adapter->ena_dev; 3238 bool readless_supported; 3239 uint32_t aenq_groups; 3240 int dma_width; 3241 int rc; 3242 3243 rc = ena_com_mmio_reg_read_request_init(ena_dev); 3244 if (unlikely(rc != 0)) { 3245 device_printf(pdev, "failed to init mmio read less\n"); 3246 return (rc); 3247 } 3248 3249 /* 3250 * The PCIe configuration space revision id indicate if mmio reg 3251 * read is disabled 3252 */ 3253 const int rev = PCI_REVISION(adapter->sc_pa.pa_class); 3254 readless_supported = ((rev & ENA_MMIO_DISABLE_REG_READ) == 0); 3255 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 3256 3257 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 3258 if (unlikely(rc != 0)) { 3259 device_printf(pdev, "Can not reset device\n"); 3260 goto err_mmio_read_less; 3261 } 3262 3263 rc = ena_com_validate_version(ena_dev); 3264 if (unlikely(rc != 0)) { 3265 device_printf(pdev, "device version is too low\n"); 3266 goto err_mmio_read_less; 3267 } 3268 3269 dma_width = ena_com_get_dma_width(ena_dev); 3270 if (unlikely(dma_width < 0)) { 3271 device_printf(pdev, "Invalid dma width value %d", dma_width); 3272 rc = dma_width; 3273 goto err_mmio_read_less; 3274 } 3275 adapter->dma_width = dma_width; 3276 3277 /* ENA admin level init */ 3278 rc = ena_com_admin_init(ena_dev, &aenq_handlers, true); 3279 if (unlikely(rc != 0)) { 3280 device_printf(pdev, 3281 "Can not initialize ena admin queue with device\n"); 3282 goto err_mmio_read_less; 3283 } 3284 3285 /* 3286 * To enable the msix interrupts the driver needs to know the number 3287 * of queues. So the driver uses polling mode to retrieve this 3288 * information 3289 */ 3290 ena_com_set_admin_polling_mode(ena_dev, true); 3291 3292 ena_config_host_info(ena_dev); 3293 3294 /* Get Device Attributes */ 3295 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 3296 if (unlikely(rc != 0)) { 3297 device_printf(pdev, 3298 "Cannot get attribute for ena device rc: %d\n", rc); 3299 goto err_admin_init; 3300 } 3301 3302 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | BIT(ENA_ADMIN_KEEP_ALIVE); 3303 3304 aenq_groups &= get_feat_ctx->aenq.supported_groups; 3305 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 3306 if (unlikely(rc != 0)) { 3307 device_printf(pdev, "Cannot configure aenq groups rc: %d\n", rc); 3308 goto err_admin_init; 3309 } 3310 3311 *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 3312 3313 return (0); 3314 3315 err_admin_init: 3316 ena_com_delete_host_info(ena_dev); 3317 ena_com_admin_destroy(ena_dev); 3318 err_mmio_read_less: 3319 ena_com_mmio_reg_read_request_destroy(ena_dev); 3320 3321 return (rc); 3322 } 3323 3324 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter, 3325 int io_vectors) 3326 { 3327 struct ena_com_dev *ena_dev = adapter->ena_dev; 3328 int rc; 3329 3330 rc = ena_enable_msix(adapter); 3331 if (unlikely(rc != 0)) { 3332 device_printf(adapter->pdev, "Error with MSI-X enablement\n"); 3333 return (rc); 3334 } 3335 3336 rc = ena_request_mgmnt_irq(adapter); 3337 if (unlikely(rc != 0)) { 3338 device_printf(adapter->pdev, "Cannot setup mgmnt queue intr\n"); 3339 goto err_disable_msix; 3340 } 3341 3342 ena_com_set_admin_polling_mode(ena_dev, false); 3343 3344 ena_com_admin_aenq_enable(ena_dev); 3345 3346 return (0); 3347 3348 err_disable_msix: 3349 ena_disable_msix(adapter); 3350 3351 return (rc); 3352 } 3353 3354 /* Function called on ENA_ADMIN_KEEP_ALIVE event */ 3355 static void ena_keep_alive_wd(void *adapter_data, 3356 struct ena_admin_aenq_entry *aenq_e) 3357 { 3358 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3359 struct ena_admin_aenq_keep_alive_desc *desc; 3360 uint64_t rx_drops, old; 3361 3362 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 3363 3364 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 3365 old = adapter->hw_stats.rx_drops.ev_count; 3366 if (rx_drops > old) { 3367 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops - old); 3368 if_statadd(adapter->ifp, if_iqdrops, rx_drops - old); 3369 } 3370 3371 atomic_store_release(&adapter->keep_alive_timestamp, getsbinuptime()); 3372 } 3373 3374 /* Check for keep alive expiration */ 3375 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 3376 { 3377 sbintime_t timestamp, time; 3378 3379 if (adapter->wd_active == 0) 3380 return; 3381 3382 if (likely(adapter->keep_alive_timeout == 0)) 3383 return; 3384 3385 timestamp = atomic_load_acquire(&adapter->keep_alive_timestamp); 3386 3387 time = getsbinuptime() - timestamp; 3388 if (unlikely(time > adapter->keep_alive_timeout)) { 3389 device_printf(adapter->pdev, 3390 "Keep alive watchdog timeout.\n"); 3391 counter_u64_add(adapter->dev_stats.wd_expired, 1); 3392 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; 3393 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 3394 } 3395 } 3396 3397 /* Check if admin queue is enabled */ 3398 static void check_for_admin_com_state(struct ena_adapter *adapter) 3399 { 3400 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == 3401 false)) { 3402 device_printf(adapter->pdev, 3403 "ENA admin queue is not in running state!\n"); 3404 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); 3405 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; 3406 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 3407 } 3408 } 3409 3410 static int 3411 check_missing_comp_in_queue(struct ena_adapter *adapter, 3412 struct ena_ring *tx_ring) 3413 { 3414 struct bintime curtime, time; 3415 struct ena_tx_buffer *tx_buf; 3416 uint32_t missed_tx = 0; 3417 int i; 3418 3419 getbinuptime(&curtime); 3420 3421 for (i = 0; i < tx_ring->ring_size; i++) { 3422 tx_buf = &tx_ring->tx_buffer_info[i]; 3423 3424 if (bintime_isset(&tx_buf->timestamp) == 0) 3425 continue; 3426 3427 time = curtime; 3428 bintime_sub(&time, &tx_buf->timestamp); 3429 3430 /* Check again if packet is still waiting */ 3431 if (unlikely(bttosbt(time) > adapter->missing_tx_timeout)) { 3432 3433 if (!tx_buf->print_once) 3434 ena_trace(ENA_WARNING, "Found a Tx that wasn't " 3435 "completed on time, qid %d, index %d.\n", 3436 tx_ring->qid, i); 3437 3438 tx_buf->print_once = true; 3439 missed_tx++; 3440 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, 1); 3441 3442 if (unlikely(missed_tx > 3443 adapter->missing_tx_threshold)) { 3444 device_printf(adapter->pdev, 3445 "The number of lost tx completion " 3446 "is above the threshold (%d > %d). " 3447 "Reset the device\n", 3448 missed_tx, adapter->missing_tx_threshold); 3449 adapter->reset_reason = 3450 ENA_REGS_RESET_MISS_TX_CMPL; 3451 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, 3452 adapter); 3453 return (EIO); 3454 } 3455 } 3456 } 3457 3458 return (0); 3459 } 3460 3461 /* 3462 * Check for TX which were not completed on time. 3463 * Timeout is defined by "missing_tx_timeout". 3464 * Reset will be performed if number of incompleted 3465 * transactions exceeds "missing_tx_threshold". 3466 */ 3467 static void 3468 check_for_missing_tx_completions(struct ena_adapter *adapter) 3469 { 3470 struct ena_ring *tx_ring; 3471 int i, budget, rc; 3472 3473 /* Make sure the driver doesn't turn the device in other process */ 3474 rmb(); 3475 3476 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3477 return; 3478 3479 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3480 return; 3481 3482 if (adapter->missing_tx_timeout == 0) 3483 return; 3484 3485 budget = adapter->missing_tx_max_queues; 3486 3487 for (i = adapter->next_monitored_tx_qid; i < adapter->num_queues; i++) { 3488 tx_ring = &adapter->tx_ring[i]; 3489 3490 rc = check_missing_comp_in_queue(adapter, tx_ring); 3491 if (unlikely(rc != 0)) 3492 return; 3493 3494 budget--; 3495 if (budget == 0) { 3496 i++; 3497 break; 3498 } 3499 } 3500 3501 adapter->next_monitored_tx_qid = i % adapter->num_queues; 3502 } 3503 3504 /* trigger deferred rx cleanup after 2 consecutive detections */ 3505 #define EMPTY_RX_REFILL 2 3506 /* For the rare case where the device runs out of Rx descriptors and the 3507 * msix handler failed to refill new Rx descriptors (due to a lack of memory 3508 * for example). 3509 * This case will lead to a deadlock: 3510 * The device won't send interrupts since all the new Rx packets will be dropped 3511 * The msix handler won't allocate new Rx descriptors so the device won't be 3512 * able to send new packets. 3513 * 3514 * When such a situation is detected - execute rx cleanup task in another thread 3515 */ 3516 static void 3517 check_for_empty_rx_ring(struct ena_adapter *adapter) 3518 { 3519 struct ena_ring *rx_ring; 3520 int i, refill_required; 3521 3522 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3523 return; 3524 3525 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3526 return; 3527 3528 for (i = 0; i < adapter->num_queues; i++) { 3529 rx_ring = &adapter->rx_ring[i]; 3530 3531 refill_required = ena_com_free_desc(rx_ring->ena_com_io_sq); 3532 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { 3533 rx_ring->empty_rx_queue++; 3534 3535 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { 3536 counter_u64_add(rx_ring->rx_stats.empty_rx_ring, 3537 1); 3538 3539 device_printf(adapter->pdev, 3540 "trigger refill for ring %d\n", i); 3541 3542 ENA_RING_MTX_LOCK(rx_ring); 3543 if (rx_ring->stopping) { 3544 ENA_RING_MTX_UNLOCK(rx_ring); 3545 return; 3546 } 3547 if (atomic_cas_uint(&rx_ring->task_pending, 0, 1) == 0) 3548 workqueue_enqueue(rx_ring->cleanup_tq, 3549 &rx_ring->cleanup_task, curcpu()); 3550 ENA_RING_MTX_UNLOCK(rx_ring); 3551 rx_ring->empty_rx_queue = 0; 3552 } 3553 } else { 3554 rx_ring->empty_rx_queue = 0; 3555 } 3556 } 3557 } 3558 3559 static void 3560 ena_timer_service(void *data) 3561 { 3562 struct ena_adapter *adapter = (struct ena_adapter *)data; 3563 struct ena_admin_host_info *host_info = 3564 adapter->ena_dev->host_attr.host_info; 3565 3566 check_for_missing_keep_alive(adapter); 3567 3568 check_for_admin_com_state(adapter); 3569 3570 check_for_missing_tx_completions(adapter); 3571 3572 check_for_empty_rx_ring(adapter); 3573 3574 if (host_info != NULL) 3575 ena_update_host_info(host_info, adapter->ifp); 3576 3577 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3578 device_printf(adapter->pdev, "Trigger reset is on\n"); 3579 workqueue_enqueue(adapter->reset_tq, &adapter->reset_task, 3580 curcpu()); 3581 return; 3582 } 3583 3584 /* 3585 * Schedule another timeout one second from now. 3586 */ 3587 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))) 3588 callout_schedule(&adapter->timer_service, hz); 3589 } 3590 3591 static void 3592 ena_reset_task(struct work *wk, void *arg) 3593 { 3594 struct ena_com_dev_get_features_ctx get_feat_ctx; 3595 struct ena_adapter *adapter = (struct ena_adapter *)arg; 3596 struct ena_com_dev *ena_dev = adapter->ena_dev; 3597 bool dev_up; 3598 int rc; 3599 3600 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3601 device_printf(adapter->pdev, 3602 "device reset scheduled but trigger_reset is off\n"); 3603 return; 3604 } 3605 3606 ENA_CORE_MTX_LOCK(adapter); 3607 3608 callout_halt(&adapter->timer_service, NULL); 3609 3610 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 3611 3612 ena_com_set_admin_running_state(ena_dev, false); 3613 ena_down(adapter); 3614 ena_free_mgmnt_irq(adapter); 3615 ena_disable_msix(adapter); 3616 ena_com_abort_admin_commands(ena_dev); 3617 ena_com_wait_for_abort_completion(ena_dev); 3618 ena_com_admin_destroy(ena_dev); 3619 ena_com_mmio_reg_read_request_destroy(ena_dev); 3620 3621 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3622 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 3623 3624 /* Finished destroy part. Restart the device */ 3625 rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx, 3626 &adapter->wd_active); 3627 if (unlikely(rc != 0)) { 3628 device_printf(adapter->pdev, 3629 "ENA device init failed! (err: %d)\n", rc); 3630 goto err_dev_free; 3631 } 3632 3633 /* XXX dealloc and realloc MSI-X, probably a waste */ 3634 rc = ena_enable_msix_and_set_admin_interrupts(adapter, 3635 adapter->num_queues); 3636 if (unlikely(rc != 0)) { 3637 device_printf(adapter->pdev, "Enable MSI-X failed\n"); 3638 goto err_com_free; 3639 } 3640 3641 /* If the interface was up before the reset bring it up */ 3642 if (dev_up) { 3643 rc = ena_up(adapter); 3644 if (unlikely(rc != 0)) { 3645 device_printf(adapter->pdev, 3646 "Failed to create I/O queues\n"); 3647 goto err_msix_free; 3648 } 3649 } 3650 3651 ENA_CORE_MTX_UNLOCK(adapter); 3652 3653 return; 3654 3655 err_msix_free: 3656 ena_free_mgmnt_irq(adapter); 3657 ena_disable_msix(adapter); 3658 err_com_free: 3659 ena_com_admin_destroy(ena_dev); 3660 err_dev_free: 3661 device_printf(adapter->pdev, "ENA reset failed!\n"); 3662 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3663 ENA_CORE_MTX_UNLOCK(adapter); 3664 } 3665 3666 /** 3667 * ena_attach - Device Initialization Routine 3668 * @pdev: device information struct 3669 * 3670 * Returns 0 on success, otherwise on failure. 3671 * 3672 * ena_attach initializes an adapter identified by a device structure. 3673 * The OS initialization, configuring of the adapter private structure, 3674 * and a hardware reset occur. 3675 **/ 3676 static void 3677 ena_attach(device_t parent, device_t self, void *aux) 3678 { 3679 struct pci_attach_args *pa = aux; 3680 struct ena_com_dev_get_features_ctx get_feat_ctx; 3681 static int version_printed; 3682 struct ena_adapter *adapter = device_private(self); 3683 struct ena_com_dev *ena_dev = NULL; 3684 uint16_t tx_sgl_size = 0; 3685 uint16_t rx_sgl_size = 0; 3686 pcireg_t reg; 3687 int io_queue_num; 3688 int queue_size; 3689 int rc; 3690 3691 adapter->pdev = self; 3692 adapter->ifp = &adapter->sc_ec.ec_if; 3693 adapter->sc_pa = *pa; /* used after attach for adapter reset too */ 3694 3695 if (pci_dma64_available(pa)) 3696 adapter->sc_dmat = pa->pa_dmat64; 3697 else 3698 adapter->sc_dmat = pa->pa_dmat; 3699 3700 pci_aprint_devinfo(pa, NULL); 3701 3702 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 3703 if ((reg & PCI_COMMAND_MASTER_ENABLE) == 0) { 3704 reg |= PCI_COMMAND_MASTER_ENABLE; 3705 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, reg); 3706 } 3707 3708 mutex_init(&adapter->global_mtx, MUTEX_DEFAULT, IPL_SOFTNET); 3709 3710 /* Set up the timer service */ 3711 adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO; 3712 adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO; 3713 adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES; 3714 adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD; 3715 3716 if (version_printed++ == 0) 3717 device_printf(parent, "%s\n", ena_version); 3718 3719 rc = ena_allocate_pci_resources(pa, adapter); 3720 if (unlikely(rc != 0)) { 3721 device_printf(parent, "PCI resource allocation failed!\n"); 3722 ena_free_pci_resources(adapter); 3723 return; 3724 } 3725 3726 /* Allocate memory for ena_dev structure */ 3727 ena_dev = kmem_zalloc(sizeof(struct ena_com_dev), KM_SLEEP); 3728 3729 adapter->ena_dev = ena_dev; 3730 ena_dev->dmadev = self; 3731 ena_dev->bus = kmem_zalloc(sizeof(struct ena_bus), KM_SLEEP); 3732 3733 /* Store register resources */ 3734 ((struct ena_bus*)(ena_dev->bus))->reg_bar_t = adapter->sc_btag; 3735 ((struct ena_bus*)(ena_dev->bus))->reg_bar_h = adapter->sc_bhandle; 3736 3737 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 3738 3739 /* Device initialization */ 3740 rc = ena_device_init(adapter, self, &get_feat_ctx, &adapter->wd_active); 3741 if (unlikely(rc != 0)) { 3742 device_printf(self, "ENA device init failed! (err: %d)\n", rc); 3743 rc = ENXIO; 3744 goto err_bus_free; 3745 } 3746 3747 adapter->keep_alive_timestamp = getsbinuptime(); 3748 3749 adapter->tx_offload_cap = get_feat_ctx.offload.tx; 3750 3751 /* Set for sure that interface is not up */ 3752 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); 3753 3754 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, 3755 ETHER_ADDR_LEN); 3756 3757 /* calculate IO queue number to create */ 3758 io_queue_num = ena_calc_io_queue_num(pa, adapter, &get_feat_ctx); 3759 3760 ENA_ASSERT(io_queue_num > 0, "Invalid queue number: %d\n", 3761 io_queue_num); 3762 adapter->num_queues = io_queue_num; 3763 3764 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 3765 3766 /* calculatre ring sizes */ 3767 queue_size = ena_calc_queue_size(adapter,&tx_sgl_size, 3768 &rx_sgl_size, &get_feat_ctx); 3769 if (unlikely((queue_size <= 0) || (io_queue_num <= 0))) { 3770 rc = ENA_COM_FAULT; 3771 goto err_com_free; 3772 } 3773 3774 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3775 3776 adapter->tx_ring_size = queue_size; 3777 adapter->rx_ring_size = queue_size; 3778 3779 adapter->max_tx_sgl_size = tx_sgl_size; 3780 adapter->max_rx_sgl_size = rx_sgl_size; 3781 3782 #if 0 3783 /* set up dma tags for rx and tx buffers */ 3784 rc = ena_setup_tx_dma_tag(adapter); 3785 if (unlikely(rc != 0)) { 3786 device_printf(self, "Failed to create TX DMA tag\n"); 3787 goto err_com_free; 3788 } 3789 3790 rc = ena_setup_rx_dma_tag(adapter); 3791 if (unlikely(rc != 0)) { 3792 device_printf(self, "Failed to create RX DMA tag\n"); 3793 goto err_tx_tag_free; 3794 } 3795 #endif 3796 3797 /* initialize rings basic information */ 3798 device_printf(self, "initialize %d io queues\n", io_queue_num); 3799 ena_init_io_rings(adapter); 3800 3801 /* setup network interface */ 3802 rc = ena_setup_ifnet(self, adapter, &get_feat_ctx); 3803 if (unlikely(rc != 0)) { 3804 device_printf(self, "Error with network interface setup\n"); 3805 goto err_io_free; 3806 } 3807 3808 rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num); 3809 if (unlikely(rc != 0)) { 3810 device_printf(self, 3811 "Failed to enable and set the admin interrupts\n"); 3812 goto err_ifp_free; 3813 } 3814 3815 callout_init(&adapter->timer_service, CALLOUT_MPSAFE); 3816 callout_setfunc(&adapter->timer_service, ena_timer_service, adapter); 3817 3818 /* Initialize reset task queue */ 3819 rc = workqueue_create(&adapter->reset_tq, "ena_reset_enq", 3820 ena_reset_task, adapter, 0, IPL_NET, WQ_PERCPU | WQ_MPSAFE); 3821 if (unlikely(rc != 0)) { 3822 ena_trace(ENA_ALERT, 3823 "Unable to create workqueue for reset task\n"); 3824 goto err_ifp_free; 3825 } 3826 3827 /* Initialize statistics */ 3828 ena_alloc_counters_dev(adapter, &adapter->dev_stats, io_queue_num); 3829 ena_alloc_counters_hwstats(adapter, &adapter->hw_stats, io_queue_num); 3830 #if 0 3831 ena_sysctl_add_nodes(adapter); 3832 #endif 3833 3834 config_interrupts(self, ena_rss_init_default); 3835 3836 /* Tell the stack that the interface is not active */ 3837 if_setdrvflagbits(adapter->ifp, IFF_OACTIVE, IFF_RUNNING); 3838 ena_set_stopping_flag(adapter, false); 3839 3840 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3841 return; 3842 3843 err_ifp_free: 3844 if_detach(adapter->ifp); 3845 err_io_free: 3846 ena_free_all_io_rings_resources(adapter); 3847 #if 0 3848 ena_free_rx_dma_tag(adapter); 3849 err_tx_tag_free: 3850 ena_free_tx_dma_tag(adapter); 3851 #endif 3852 err_com_free: 3853 ena_com_admin_destroy(ena_dev); 3854 ena_com_delete_host_info(ena_dev); 3855 ena_com_mmio_reg_read_request_destroy(ena_dev); 3856 err_bus_free: 3857 kmem_free(ena_dev->bus, sizeof(struct ena_bus)); 3858 kmem_free(ena_dev, sizeof(struct ena_com_dev)); 3859 ena_free_pci_resources(adapter); 3860 } 3861 3862 /** 3863 * ena_detach - Device Removal Routine 3864 * @pdev: device information struct 3865 * 3866 * ena_detach is called by the device subsystem to alert the driver 3867 * that it should release a PCI device. 3868 **/ 3869 static int 3870 ena_detach(device_t pdev, int flags) 3871 { 3872 struct ena_adapter *adapter = device_private(pdev); 3873 struct ena_com_dev *ena_dev = adapter->ena_dev; 3874 #if 0 3875 int rc; 3876 #endif 3877 3878 /* Make sure VLANS are not using driver */ 3879 if (VLAN_ATTACHED(&adapter->sc_ec)) { 3880 device_printf(adapter->pdev ,"VLAN is in use, detach first\n"); 3881 return (EBUSY); 3882 } 3883 3884 ena_com_set_admin_running_state(ena_dev, false); 3885 ENA_CORE_MTX_LOCK(adapter); 3886 ena_down(adapter); 3887 ENA_CORE_MTX_UNLOCK(adapter); 3888 3889 /* Free reset task and callout */ 3890 callout_halt(&adapter->timer_service, NULL); 3891 callout_destroy(&adapter->timer_service); 3892 workqueue_wait(adapter->reset_tq, &adapter->reset_task); 3893 workqueue_destroy(adapter->reset_tq); 3894 adapter->reset_tq = NULL; 3895 3896 ena_free_all_io_rings_resources(adapter); 3897 3898 if (likely(adapter->rss_support)) 3899 ena_com_rss_destroy(ena_dev); 3900 3901 #if 0 3902 rc = ena_free_rx_dma_tag(adapter); 3903 if (unlikely(rc != 0)) 3904 device_printf(adapter->pdev, 3905 "Unmapped RX DMA tag associations\n"); 3906 3907 rc = ena_free_tx_dma_tag(adapter); 3908 if (unlikely(rc != 0)) 3909 device_printf(adapter->pdev, 3910 "Unmapped TX DMA tag associations\n"); 3911 #endif 3912 3913 /* Reset the device only if the device is running. */ 3914 if (ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) 3915 ena_com_dev_reset(ena_dev, adapter->reset_reason); 3916 3917 ena_com_delete_host_info(ena_dev); 3918 3919 ena_free_irqs(adapter); 3920 3921 ena_com_abort_admin_commands(ena_dev); 3922 3923 ena_com_wait_for_abort_completion(ena_dev); 3924 3925 ena_com_admin_destroy(ena_dev); 3926 3927 ena_com_mmio_reg_read_request_destroy(ena_dev); 3928 3929 ena_free_pci_resources(adapter); 3930 3931 ena_free_counters((struct evcnt *)&adapter->hw_stats, 3932 sizeof(struct ena_hw_stats), 3933 offsetof(struct ena_hw_stats, rx_packets)); 3934 ena_free_counters((struct evcnt *)&adapter->dev_stats, 3935 sizeof(struct ena_stats_dev), 3936 offsetof(struct ena_stats_dev, wd_expired)); 3937 3938 if (adapter->ifp != NULL) { 3939 ether_ifdetach(adapter->ifp); 3940 if_detach(adapter->ifp); 3941 } 3942 ifmedia_fini(&adapter->media); 3943 3944 mutex_destroy(&adapter->global_mtx); 3945 3946 if (ena_dev->bus != NULL) 3947 kmem_free(ena_dev->bus, sizeof(struct ena_bus)); 3948 3949 if (ena_dev != NULL) 3950 kmem_free(ena_dev, sizeof(struct ena_com_dev)); 3951 3952 return 0; 3953 } 3954 3955 /****************************************************************************** 3956 ******************************** AENQ Handlers ******************************* 3957 *****************************************************************************/ 3958 /** 3959 * ena_update_on_link_change: 3960 * Notify the network interface about the change in link status 3961 **/ 3962 static void 3963 ena_update_on_link_change(void *adapter_data, 3964 struct ena_admin_aenq_entry *aenq_e) 3965 { 3966 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3967 struct ena_admin_aenq_link_change_desc *aenq_desc; 3968 int status; 3969 struct ifnet *ifp; 3970 3971 aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 3972 ifp = adapter->ifp; 3973 status = aenq_desc->flags & 3974 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; 3975 3976 if (status != 0) { 3977 device_printf(adapter->pdev, "link is UP\n"); 3978 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3979 if_link_state_change(ifp, LINK_STATE_UP); 3980 } else { 3981 device_printf(adapter->pdev, "link is DOWN\n"); 3982 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3983 if_link_state_change(ifp, LINK_STATE_DOWN); 3984 } 3985 } 3986 3987 /** 3988 * This handler will called for unknown event group or unimplemented handlers 3989 **/ 3990 static void 3991 unimplemented_aenq_handler(void *data, 3992 struct ena_admin_aenq_entry *aenq_e) 3993 { 3994 return; 3995 } 3996 3997 static struct ena_aenq_handlers aenq_handlers = { 3998 .handlers = { 3999 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 4000 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd, 4001 }, 4002 .unimplemented_handler = unimplemented_aenq_handler 4003 }; 4004 4005 #ifdef __FreeBSD__ 4006 /********************************************************************* 4007 * FreeBSD Device Interface Entry Points 4008 *********************************************************************/ 4009 4010 static device_method_t ena_methods[] = { 4011 /* Device interface */ 4012 DEVMETHOD(device_probe, ena_probe), 4013 DEVMETHOD(device_attach, ena_attach), 4014 DEVMETHOD(device_detach, ena_detach), 4015 DEVMETHOD_END 4016 }; 4017 4018 static driver_t ena_driver = { 4019 "ena", ena_methods, sizeof(struct ena_adapter), 4020 }; 4021 4022 devclass_t ena_devclass; 4023 DRIVER_MODULE(ena, pci, ena_driver, ena_devclass, 0, 0); 4024 MODULE_DEPEND(ena, pci, 1, 1, 1); 4025 MODULE_DEPEND(ena, ether, 1, 1, 1); 4026 4027 /*********************************************************************/ 4028 #endif /* __FreeBSD__ */ 4029 4030 #ifdef __NetBSD__ 4031 CFATTACH_DECL_NEW(ena, sizeof(struct ena_adapter), ena_probe, ena_attach, 4032 ena_detach, NULL); 4033 #endif /* __NetBSD */ 4034