1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev_driver.h> 36 #include <rte_ethdev_pci.h> 37 #include <rte_tcp.h> 38 #include <rte_atomic.h> 39 #include <rte_dev.h> 40 #include <rte_errno.h> 41 #include <rte_version.h> 42 #include <rte_eal_memconfig.h> 43 #include <rte_net.h> 44 45 #include "ena_ethdev.h" 46 #include "ena_logs.h" 47 #include "ena_platform.h" 48 #include "ena_com.h" 49 #include "ena_eth_com.h" 50 51 #include <ena_common_defs.h> 52 #include <ena_regs_defs.h> 53 #include <ena_admin_defs.h> 54 #include <ena_eth_io_defs.h> 55 56 #define DRV_MODULE_VER_MAJOR 1 57 #define DRV_MODULE_VER_MINOR 1 58 #define DRV_MODULE_VER_SUBMINOR 0 59 60 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 61 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 62 /*reverse version of ENA_IO_RXQ_IDX*/ 63 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 64 65 /* While processing submitted and completed descriptors (rx and tx path 66 * respectively) in a loop it is desired to: 67 * - perform batch submissions while populating sumbissmion queue 68 * - avoid blocking transmission of other packets during cleanup phase 69 * Hence the utilization ratio of 1/8 of a queue size. 70 */ 71 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 72 73 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 74 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 75 76 #define GET_L4_HDR_LEN(mbuf) \ 77 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 78 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 79 80 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 81 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 82 #define ENA_HASH_KEY_SIZE 40 83 #define ENA_ETH_SS_STATS 0xFF 84 #define ETH_GSTRING_LEN 32 85 86 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 87 88 #define ENA_MAX_RING_DESC ENA_DEFAULT_RING_SIZE 89 #define ENA_MIN_RING_DESC 128 90 91 enum ethtool_stringset { 92 ETH_SS_TEST = 0, 93 ETH_SS_STATS, 94 }; 95 96 struct ena_stats { 97 char name[ETH_GSTRING_LEN]; 98 int stat_offset; 99 }; 100 101 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 102 .name = #stat, \ 103 .stat_offset = offsetof(struct ena_com_stats_admin, stat) \ 104 } 105 106 #define ENA_STAT_ENTRY(stat, stat_type) { \ 107 .name = #stat, \ 108 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 109 } 110 111 #define ENA_STAT_RX_ENTRY(stat) \ 112 ENA_STAT_ENTRY(stat, rx) 113 114 #define ENA_STAT_TX_ENTRY(stat) \ 115 ENA_STAT_ENTRY(stat, tx) 116 117 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 118 ENA_STAT_ENTRY(stat, dev) 119 120 /* 121 * Each rte_memzone should have unique name. 122 * To satisfy it, count number of allocation and add it to name. 123 */ 124 uint32_t ena_alloc_cnt; 125 126 static const struct ena_stats ena_stats_global_strings[] = { 127 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 128 ENA_STAT_GLOBAL_ENTRY(io_suspend), 129 ENA_STAT_GLOBAL_ENTRY(io_resume), 130 ENA_STAT_GLOBAL_ENTRY(wd_expired), 131 ENA_STAT_GLOBAL_ENTRY(interface_up), 132 ENA_STAT_GLOBAL_ENTRY(interface_down), 133 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 134 }; 135 136 static const struct ena_stats ena_stats_tx_strings[] = { 137 ENA_STAT_TX_ENTRY(cnt), 138 ENA_STAT_TX_ENTRY(bytes), 139 ENA_STAT_TX_ENTRY(queue_stop), 140 ENA_STAT_TX_ENTRY(queue_wakeup), 141 ENA_STAT_TX_ENTRY(dma_mapping_err), 142 ENA_STAT_TX_ENTRY(linearize), 143 ENA_STAT_TX_ENTRY(linearize_failed), 144 ENA_STAT_TX_ENTRY(tx_poll), 145 ENA_STAT_TX_ENTRY(doorbells), 146 ENA_STAT_TX_ENTRY(prepare_ctx_err), 147 ENA_STAT_TX_ENTRY(missing_tx_comp), 148 ENA_STAT_TX_ENTRY(bad_req_id), 149 }; 150 151 static const struct ena_stats ena_stats_rx_strings[] = { 152 ENA_STAT_RX_ENTRY(cnt), 153 ENA_STAT_RX_ENTRY(bytes), 154 ENA_STAT_RX_ENTRY(refil_partial), 155 ENA_STAT_RX_ENTRY(bad_csum), 156 ENA_STAT_RX_ENTRY(page_alloc_fail), 157 ENA_STAT_RX_ENTRY(skb_alloc_fail), 158 ENA_STAT_RX_ENTRY(dma_mapping_err), 159 ENA_STAT_RX_ENTRY(bad_desc_num), 160 ENA_STAT_RX_ENTRY(small_copy_len_pkt), 161 }; 162 163 static const struct ena_stats ena_stats_ena_com_strings[] = { 164 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 165 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 166 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 167 ENA_STAT_ENA_COM_ENTRY(out_of_space), 168 ENA_STAT_ENA_COM_ENTRY(no_completion), 169 }; 170 171 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 172 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 173 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 174 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 175 176 #define QUEUE_OFFLOADS (DEV_TX_OFFLOAD_TCP_CKSUM |\ 177 DEV_TX_OFFLOAD_UDP_CKSUM |\ 178 DEV_TX_OFFLOAD_IPV4_CKSUM |\ 179 DEV_TX_OFFLOAD_TCP_TSO) 180 #define MBUF_OFFLOADS (PKT_TX_L4_MASK |\ 181 PKT_TX_IP_CKSUM |\ 182 PKT_TX_TCP_SEG) 183 184 /** Vendor ID used by Amazon devices */ 185 #define PCI_VENDOR_ID_AMAZON 0x1D0F 186 /** Amazon devices */ 187 #define PCI_DEVICE_ID_ENA_VF 0xEC20 188 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 189 190 #define ENA_TX_OFFLOAD_MASK (\ 191 PKT_TX_L4_MASK | \ 192 PKT_TX_IP_CKSUM | \ 193 PKT_TX_TCP_SEG) 194 195 #define ENA_TX_OFFLOAD_NOTSUP_MASK \ 196 (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) 197 198 int ena_logtype_init; 199 int ena_logtype_driver; 200 201 static const struct rte_pci_id pci_id_ena_map[] = { 202 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 203 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, 204 { .device_id = 0 }, 205 }; 206 207 static struct ena_aenq_handlers aenq_handlers; 208 209 static int ena_device_init(struct ena_com_dev *ena_dev, 210 struct ena_com_dev_get_features_ctx *get_feat_ctx, 211 bool *wd_state); 212 static int ena_dev_configure(struct rte_eth_dev *dev); 213 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 214 uint16_t nb_pkts); 215 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 216 uint16_t nb_pkts); 217 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 218 uint16_t nb_desc, unsigned int socket_id, 219 const struct rte_eth_txconf *tx_conf); 220 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 221 uint16_t nb_desc, unsigned int socket_id, 222 const struct rte_eth_rxconf *rx_conf, 223 struct rte_mempool *mp); 224 static uint16_t eth_ena_recv_pkts(void *rx_queue, 225 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 226 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 227 static void ena_init_rings(struct ena_adapter *adapter); 228 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 229 static int ena_start(struct rte_eth_dev *dev); 230 static void ena_stop(struct rte_eth_dev *dev); 231 static void ena_close(struct rte_eth_dev *dev); 232 static int ena_dev_reset(struct rte_eth_dev *dev); 233 static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 234 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 235 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 236 static void ena_rx_queue_release(void *queue); 237 static void ena_tx_queue_release(void *queue); 238 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 239 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 240 static int ena_link_update(struct rte_eth_dev *dev, 241 int wait_to_complete); 242 static int ena_queue_restart(struct ena_ring *ring); 243 static int ena_queue_restart_all(struct rte_eth_dev *dev, 244 enum ena_ring_type ring_type); 245 static void ena_stats_restart(struct rte_eth_dev *dev); 246 static void ena_infos_get(struct rte_eth_dev *dev, 247 struct rte_eth_dev_info *dev_info); 248 static int ena_rss_reta_update(struct rte_eth_dev *dev, 249 struct rte_eth_rss_reta_entry64 *reta_conf, 250 uint16_t reta_size); 251 static int ena_rss_reta_query(struct rte_eth_dev *dev, 252 struct rte_eth_rss_reta_entry64 *reta_conf, 253 uint16_t reta_size); 254 static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); 255 static void ena_interrupt_handler_rte(void *cb_arg); 256 static void ena_timer_wd_callback(struct rte_timer *timer, void *arg); 257 258 static const struct eth_dev_ops ena_dev_ops = { 259 .dev_configure = ena_dev_configure, 260 .dev_infos_get = ena_infos_get, 261 .rx_queue_setup = ena_rx_queue_setup, 262 .tx_queue_setup = ena_tx_queue_setup, 263 .dev_start = ena_start, 264 .dev_stop = ena_stop, 265 .link_update = ena_link_update, 266 .stats_get = ena_stats_get, 267 .mtu_set = ena_mtu_set, 268 .rx_queue_release = ena_rx_queue_release, 269 .tx_queue_release = ena_tx_queue_release, 270 .dev_close = ena_close, 271 .dev_reset = ena_dev_reset, 272 .reta_update = ena_rss_reta_update, 273 .reta_query = ena_rss_reta_query, 274 }; 275 276 #define NUMA_NO_NODE SOCKET_ID_ANY 277 278 static inline int ena_cpu_to_node(int cpu) 279 { 280 struct rte_config *config = rte_eal_get_configuration(); 281 struct rte_fbarray *arr = &config->mem_config->memzones; 282 const struct rte_memzone *mz; 283 284 if (unlikely(cpu >= RTE_MAX_MEMZONE)) 285 return NUMA_NO_NODE; 286 287 mz = rte_fbarray_get(arr, cpu); 288 289 return mz->socket_id; 290 } 291 292 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 293 struct ena_com_rx_ctx *ena_rx_ctx) 294 { 295 uint64_t ol_flags = 0; 296 uint32_t packet_type = 0; 297 298 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 299 packet_type |= RTE_PTYPE_L4_TCP; 300 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 301 packet_type |= RTE_PTYPE_L4_UDP; 302 303 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 304 packet_type |= RTE_PTYPE_L3_IPV4; 305 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 306 packet_type |= RTE_PTYPE_L3_IPV6; 307 308 if (unlikely(ena_rx_ctx->l4_csum_err)) 309 ol_flags |= PKT_RX_L4_CKSUM_BAD; 310 if (unlikely(ena_rx_ctx->l3_csum_err)) 311 ol_flags |= PKT_RX_IP_CKSUM_BAD; 312 313 mbuf->ol_flags = ol_flags; 314 mbuf->packet_type = packet_type; 315 } 316 317 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 318 struct ena_com_tx_ctx *ena_tx_ctx, 319 uint64_t queue_offloads) 320 { 321 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 322 323 if ((mbuf->ol_flags & MBUF_OFFLOADS) && 324 (queue_offloads & QUEUE_OFFLOADS)) { 325 /* check if TSO is required */ 326 if ((mbuf->ol_flags & PKT_TX_TCP_SEG) && 327 (queue_offloads & DEV_TX_OFFLOAD_TCP_TSO)) { 328 ena_tx_ctx->tso_enable = true; 329 330 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 331 } 332 333 /* check if L3 checksum is needed */ 334 if ((mbuf->ol_flags & PKT_TX_IP_CKSUM) && 335 (queue_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)) 336 ena_tx_ctx->l3_csum_enable = true; 337 338 if (mbuf->ol_flags & PKT_TX_IPV6) { 339 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 340 } else { 341 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 342 343 /* set don't fragment (DF) flag */ 344 if (mbuf->packet_type & 345 (RTE_PTYPE_L4_NONFRAG 346 | RTE_PTYPE_INNER_L4_NONFRAG)) 347 ena_tx_ctx->df = true; 348 } 349 350 /* check if L4 checksum is needed */ 351 if ((mbuf->ol_flags & PKT_TX_TCP_CKSUM) && 352 (queue_offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) { 353 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 354 ena_tx_ctx->l4_csum_enable = true; 355 } else if ((mbuf->ol_flags & PKT_TX_UDP_CKSUM) && 356 (queue_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) { 357 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 358 ena_tx_ctx->l4_csum_enable = true; 359 } else { 360 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 361 ena_tx_ctx->l4_csum_enable = false; 362 } 363 364 ena_meta->mss = mbuf->tso_segsz; 365 ena_meta->l3_hdr_len = mbuf->l3_len; 366 ena_meta->l3_hdr_offset = mbuf->l2_len; 367 368 ena_tx_ctx->meta_valid = true; 369 } else { 370 ena_tx_ctx->meta_valid = false; 371 } 372 } 373 374 static inline int validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id) 375 { 376 if (likely(req_id < rx_ring->ring_size)) 377 return 0; 378 379 RTE_LOG(ERR, PMD, "Invalid rx req_id: %hu\n", req_id); 380 381 rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; 382 rx_ring->adapter->trigger_reset = true; 383 384 return -EFAULT; 385 } 386 387 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) 388 { 389 struct ena_tx_buffer *tx_info = NULL; 390 391 if (likely(req_id < tx_ring->ring_size)) { 392 tx_info = &tx_ring->tx_buffer_info[req_id]; 393 if (likely(tx_info->mbuf)) 394 return 0; 395 } 396 397 if (tx_info) 398 RTE_LOG(ERR, PMD, "tx_info doesn't have valid mbuf\n"); 399 else 400 RTE_LOG(ERR, PMD, "Invalid req_id: %hu\n", req_id); 401 402 /* Trigger device reset */ 403 tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; 404 tx_ring->adapter->trigger_reset = true; 405 return -EFAULT; 406 } 407 408 static void ena_config_host_info(struct ena_com_dev *ena_dev) 409 { 410 struct ena_admin_host_info *host_info; 411 int rc; 412 413 /* Allocate only the host info */ 414 rc = ena_com_allocate_host_info(ena_dev); 415 if (rc) { 416 RTE_LOG(ERR, PMD, "Cannot allocate host info\n"); 417 return; 418 } 419 420 host_info = ena_dev->host_attr.host_info; 421 422 host_info->os_type = ENA_ADMIN_OS_DPDK; 423 host_info->kernel_ver = RTE_VERSION; 424 snprintf((char *)host_info->kernel_ver_str, 425 sizeof(host_info->kernel_ver_str), 426 "%s", rte_version()); 427 host_info->os_dist = RTE_VERSION; 428 snprintf((char *)host_info->os_dist_str, 429 sizeof(host_info->os_dist_str), 430 "%s", rte_version()); 431 host_info->driver_version = 432 (DRV_MODULE_VER_MAJOR) | 433 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 434 (DRV_MODULE_VER_SUBMINOR << 435 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 436 437 rc = ena_com_set_host_attributes(ena_dev); 438 if (rc) { 439 if (rc == -ENA_COM_UNSUPPORTED) 440 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 441 else 442 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 443 444 goto err; 445 } 446 447 return; 448 449 err: 450 ena_com_delete_host_info(ena_dev); 451 } 452 453 static int 454 ena_get_sset_count(struct rte_eth_dev *dev, int sset) 455 { 456 if (sset != ETH_SS_STATS) 457 return -EOPNOTSUPP; 458 459 /* Workaround for clang: 460 * touch internal structures to prevent 461 * compiler error 462 */ 463 ENA_TOUCH(ena_stats_global_strings); 464 ENA_TOUCH(ena_stats_tx_strings); 465 ENA_TOUCH(ena_stats_rx_strings); 466 ENA_TOUCH(ena_stats_ena_com_strings); 467 468 return dev->data->nb_tx_queues * 469 (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) + 470 ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 471 } 472 473 static void ena_config_debug_area(struct ena_adapter *adapter) 474 { 475 u32 debug_area_size; 476 int rc, ss_count; 477 478 ss_count = ena_get_sset_count(adapter->rte_dev, ETH_SS_STATS); 479 if (ss_count <= 0) { 480 RTE_LOG(ERR, PMD, "SS count is negative\n"); 481 return; 482 } 483 484 /* allocate 32 bytes for each string and 64bit for the value */ 485 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 486 487 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 488 if (rc) { 489 RTE_LOG(ERR, PMD, "Cannot allocate debug area\n"); 490 return; 491 } 492 493 rc = ena_com_set_host_attributes(&adapter->ena_dev); 494 if (rc) { 495 if (rc == -ENA_COM_UNSUPPORTED) 496 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 497 else 498 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 499 500 goto err; 501 } 502 503 return; 504 err: 505 ena_com_delete_debug_area(&adapter->ena_dev); 506 } 507 508 static void ena_close(struct rte_eth_dev *dev) 509 { 510 struct ena_adapter *adapter = 511 (struct ena_adapter *)(dev->data->dev_private); 512 513 ena_stop(dev); 514 adapter->state = ENA_ADAPTER_STATE_CLOSED; 515 516 ena_rx_queue_release_all(dev); 517 ena_tx_queue_release_all(dev); 518 } 519 520 static int 521 ena_dev_reset(struct rte_eth_dev *dev) 522 { 523 struct rte_mempool *mb_pool_rx[ENA_MAX_NUM_QUEUES]; 524 struct rte_eth_dev *eth_dev; 525 struct rte_pci_device *pci_dev; 526 struct rte_intr_handle *intr_handle; 527 struct ena_com_dev *ena_dev; 528 struct ena_com_dev_get_features_ctx get_feat_ctx; 529 struct ena_adapter *adapter; 530 int nb_queues; 531 int rc, i; 532 bool wd_state; 533 534 adapter = (struct ena_adapter *)(dev->data->dev_private); 535 ena_dev = &adapter->ena_dev; 536 eth_dev = adapter->rte_dev; 537 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 538 intr_handle = &pci_dev->intr_handle; 539 nb_queues = eth_dev->data->nb_rx_queues; 540 541 ena_com_set_admin_running_state(ena_dev, false); 542 543 rc = ena_com_dev_reset(ena_dev, adapter->reset_reason); 544 if (rc) 545 RTE_LOG(ERR, PMD, "Device reset failed\n"); 546 547 for (i = 0; i < nb_queues; i++) 548 mb_pool_rx[i] = adapter->rx_ring[i].mb_pool; 549 550 ena_rx_queue_release_all(eth_dev); 551 ena_tx_queue_release_all(eth_dev); 552 553 rte_intr_disable(intr_handle); 554 555 ena_com_abort_admin_commands(ena_dev); 556 ena_com_wait_for_abort_completion(ena_dev); 557 ena_com_admin_destroy(ena_dev); 558 ena_com_mmio_reg_read_request_destroy(ena_dev); 559 560 rc = ena_device_init(ena_dev, &get_feat_ctx, &wd_state); 561 if (rc) { 562 PMD_INIT_LOG(CRIT, "Cannot initialize device\n"); 563 return rc; 564 } 565 adapter->wd_state = wd_state; 566 567 rte_intr_enable(intr_handle); 568 ena_com_set_admin_polling_mode(ena_dev, false); 569 ena_com_admin_aenq_enable(ena_dev); 570 571 for (i = 0; i < nb_queues; ++i) 572 ena_rx_queue_setup(eth_dev, i, adapter->rx_ring_size, 0, NULL, 573 mb_pool_rx[i]); 574 575 for (i = 0; i < nb_queues; ++i) 576 ena_tx_queue_setup(eth_dev, i, adapter->tx_ring_size, 0, NULL); 577 578 adapter->trigger_reset = false; 579 580 return 0; 581 } 582 583 static int ena_rss_reta_update(struct rte_eth_dev *dev, 584 struct rte_eth_rss_reta_entry64 *reta_conf, 585 uint16_t reta_size) 586 { 587 struct ena_adapter *adapter = 588 (struct ena_adapter *)(dev->data->dev_private); 589 struct ena_com_dev *ena_dev = &adapter->ena_dev; 590 int rc, i; 591 u16 entry_value; 592 int conf_idx; 593 int idx; 594 595 if ((reta_size == 0) || (reta_conf == NULL)) 596 return -EINVAL; 597 598 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 599 RTE_LOG(WARNING, PMD, 600 "indirection table %d is bigger than supported (%d)\n", 601 reta_size, ENA_RX_RSS_TABLE_SIZE); 602 return -EINVAL; 603 } 604 605 for (i = 0 ; i < reta_size ; i++) { 606 /* each reta_conf is for 64 entries. 607 * to support 128 we use 2 conf of 64 608 */ 609 conf_idx = i / RTE_RETA_GROUP_SIZE; 610 idx = i % RTE_RETA_GROUP_SIZE; 611 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 612 entry_value = 613 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 614 615 rc = ena_com_indirect_table_fill_entry(ena_dev, 616 i, 617 entry_value); 618 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 619 RTE_LOG(ERR, PMD, 620 "Cannot fill indirect table\n"); 621 return rc; 622 } 623 } 624 } 625 626 rc = ena_com_indirect_table_set(ena_dev); 627 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 628 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 629 return rc; 630 } 631 632 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 633 __func__, reta_size, adapter->rte_dev->data->port_id); 634 635 return 0; 636 } 637 638 /* Query redirection table. */ 639 static int ena_rss_reta_query(struct rte_eth_dev *dev, 640 struct rte_eth_rss_reta_entry64 *reta_conf, 641 uint16_t reta_size) 642 { 643 struct ena_adapter *adapter = 644 (struct ena_adapter *)(dev->data->dev_private); 645 struct ena_com_dev *ena_dev = &adapter->ena_dev; 646 int rc; 647 int i; 648 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 649 int reta_conf_idx; 650 int reta_idx; 651 652 if (reta_size == 0 || reta_conf == NULL || 653 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 654 return -EINVAL; 655 656 rc = ena_com_indirect_table_get(ena_dev, indirect_table); 657 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 658 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 659 return -ENOTSUP; 660 } 661 662 for (i = 0 ; i < reta_size ; i++) { 663 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 664 reta_idx = i % RTE_RETA_GROUP_SIZE; 665 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 666 reta_conf[reta_conf_idx].reta[reta_idx] = 667 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 668 } 669 670 return 0; 671 } 672 673 static int ena_rss_init_default(struct ena_adapter *adapter) 674 { 675 struct ena_com_dev *ena_dev = &adapter->ena_dev; 676 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 677 int rc, i; 678 u32 val; 679 680 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 681 if (unlikely(rc)) { 682 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 683 goto err_rss_init; 684 } 685 686 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 687 val = i % nb_rx_queues; 688 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 689 ENA_IO_RXQ_IDX(val)); 690 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 691 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 692 goto err_fill_indir; 693 } 694 } 695 696 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 697 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 698 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 699 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 700 goto err_fill_indir; 701 } 702 703 rc = ena_com_set_default_hash_ctrl(ena_dev); 704 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 705 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 706 goto err_fill_indir; 707 } 708 709 rc = ena_com_indirect_table_set(ena_dev); 710 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 711 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 712 goto err_fill_indir; 713 } 714 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 715 adapter->rte_dev->data->port_id); 716 717 return 0; 718 719 err_fill_indir: 720 ena_com_rss_destroy(ena_dev); 721 err_rss_init: 722 723 return rc; 724 } 725 726 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 727 { 728 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 729 int nb_queues = dev->data->nb_rx_queues; 730 int i; 731 732 for (i = 0; i < nb_queues; i++) 733 ena_rx_queue_release(queues[i]); 734 } 735 736 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 737 { 738 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 739 int nb_queues = dev->data->nb_tx_queues; 740 int i; 741 742 for (i = 0; i < nb_queues; i++) 743 ena_tx_queue_release(queues[i]); 744 } 745 746 static void ena_rx_queue_release(void *queue) 747 { 748 struct ena_ring *ring = (struct ena_ring *)queue; 749 struct ena_adapter *adapter = ring->adapter; 750 int ena_qid; 751 752 ena_assert_msg(ring->configured, 753 "API violation - releasing not configured queue"); 754 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 755 "API violation"); 756 757 /* Destroy HW queue */ 758 ena_qid = ENA_IO_RXQ_IDX(ring->id); 759 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 760 761 /* Free all bufs */ 762 ena_rx_queue_release_bufs(ring); 763 764 /* Free ring resources */ 765 if (ring->rx_buffer_info) 766 rte_free(ring->rx_buffer_info); 767 ring->rx_buffer_info = NULL; 768 769 if (ring->empty_rx_reqs) 770 rte_free(ring->empty_rx_reqs); 771 ring->empty_rx_reqs = NULL; 772 773 ring->configured = 0; 774 775 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 776 ring->port_id, ring->id); 777 } 778 779 static void ena_tx_queue_release(void *queue) 780 { 781 struct ena_ring *ring = (struct ena_ring *)queue; 782 struct ena_adapter *adapter = ring->adapter; 783 int ena_qid; 784 785 ena_assert_msg(ring->configured, 786 "API violation. Releasing not configured queue"); 787 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 788 "API violation"); 789 790 /* Destroy HW queue */ 791 ena_qid = ENA_IO_TXQ_IDX(ring->id); 792 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 793 794 /* Free all bufs */ 795 ena_tx_queue_release_bufs(ring); 796 797 /* Free ring resources */ 798 if (ring->tx_buffer_info) 799 rte_free(ring->tx_buffer_info); 800 801 if (ring->empty_tx_reqs) 802 rte_free(ring->empty_tx_reqs); 803 804 ring->empty_tx_reqs = NULL; 805 ring->tx_buffer_info = NULL; 806 807 ring->configured = 0; 808 809 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 810 ring->port_id, ring->id); 811 } 812 813 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 814 { 815 unsigned int ring_mask = ring->ring_size - 1; 816 817 while (ring->next_to_clean != ring->next_to_use) { 818 struct rte_mbuf *m = 819 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 820 821 if (m) 822 rte_mbuf_raw_free(m); 823 824 ring->next_to_clean++; 825 } 826 } 827 828 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 829 { 830 unsigned int i; 831 832 for (i = 0; i < ring->ring_size; ++i) { 833 struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; 834 835 if (tx_buf->mbuf) 836 rte_pktmbuf_free(tx_buf->mbuf); 837 838 ring->next_to_clean++; 839 } 840 } 841 842 static int ena_link_update(struct rte_eth_dev *dev, 843 __rte_unused int wait_to_complete) 844 { 845 struct rte_eth_link *link = &dev->data->dev_link; 846 struct ena_adapter *adapter; 847 848 adapter = (struct ena_adapter *)(dev->data->dev_private); 849 850 link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN; 851 link->link_speed = ETH_SPEED_NUM_NONE; 852 link->link_duplex = ETH_LINK_FULL_DUPLEX; 853 854 return 0; 855 } 856 857 static int ena_queue_restart_all(struct rte_eth_dev *dev, 858 enum ena_ring_type ring_type) 859 { 860 struct ena_adapter *adapter = 861 (struct ena_adapter *)(dev->data->dev_private); 862 struct ena_ring *queues = NULL; 863 int nb_queues; 864 int i = 0; 865 int rc = 0; 866 867 if (ring_type == ENA_RING_TYPE_RX) { 868 queues = adapter->rx_ring; 869 nb_queues = dev->data->nb_rx_queues; 870 } else { 871 queues = adapter->tx_ring; 872 nb_queues = dev->data->nb_tx_queues; 873 } 874 for (i = 0; i < nb_queues; i++) { 875 if (queues[i].configured) { 876 if (ring_type == ENA_RING_TYPE_RX) { 877 ena_assert_msg( 878 dev->data->rx_queues[i] == &queues[i], 879 "Inconsistent state of rx queues\n"); 880 } else { 881 ena_assert_msg( 882 dev->data->tx_queues[i] == &queues[i], 883 "Inconsistent state of tx queues\n"); 884 } 885 886 rc = ena_queue_restart(&queues[i]); 887 888 if (rc) { 889 PMD_INIT_LOG(ERR, 890 "failed to restart queue %d type(%d)", 891 i, ring_type); 892 return rc; 893 } 894 } 895 } 896 897 return 0; 898 } 899 900 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 901 { 902 uint32_t max_frame_len = adapter->max_mtu; 903 904 if (adapter->rte_eth_dev_data->dev_conf.rxmode.offloads & 905 DEV_RX_OFFLOAD_JUMBO_FRAME) 906 max_frame_len = 907 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 908 909 return max_frame_len; 910 } 911 912 static int ena_check_valid_conf(struct ena_adapter *adapter) 913 { 914 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 915 916 if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) { 917 PMD_INIT_LOG(ERR, "Unsupported MTU of %d. " 918 "max mtu: %d, min mtu: %d\n", 919 max_frame_len, adapter->max_mtu, ENA_MIN_MTU); 920 return ENA_COM_UNSUPPORTED; 921 } 922 923 return 0; 924 } 925 926 static int 927 ena_calc_queue_size(struct ena_com_dev *ena_dev, 928 u16 *max_tx_sgl_size, 929 struct ena_com_dev_get_features_ctx *get_feat_ctx) 930 { 931 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 932 933 queue_size = RTE_MIN(queue_size, 934 get_feat_ctx->max_queues.max_cq_depth); 935 queue_size = RTE_MIN(queue_size, 936 get_feat_ctx->max_queues.max_sq_depth); 937 938 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 939 queue_size = RTE_MIN(queue_size, 940 get_feat_ctx->max_queues.max_llq_depth); 941 942 /* Round down to power of 2 */ 943 if (!rte_is_power_of_2(queue_size)) 944 queue_size = rte_align32pow2(queue_size >> 1); 945 946 if (unlikely(queue_size == 0)) { 947 PMD_INIT_LOG(ERR, "Invalid queue size"); 948 return -EFAULT; 949 } 950 951 *max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 952 get_feat_ctx->max_queues.max_packet_tx_descs); 953 954 return queue_size; 955 } 956 957 static void ena_stats_restart(struct rte_eth_dev *dev) 958 { 959 struct ena_adapter *adapter = 960 (struct ena_adapter *)(dev->data->dev_private); 961 962 rte_atomic64_init(&adapter->drv_stats->ierrors); 963 rte_atomic64_init(&adapter->drv_stats->oerrors); 964 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 965 } 966 967 static int ena_stats_get(struct rte_eth_dev *dev, 968 struct rte_eth_stats *stats) 969 { 970 struct ena_admin_basic_stats ena_stats; 971 struct ena_adapter *adapter = 972 (struct ena_adapter *)(dev->data->dev_private); 973 struct ena_com_dev *ena_dev = &adapter->ena_dev; 974 int rc; 975 976 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 977 return -ENOTSUP; 978 979 memset(&ena_stats, 0, sizeof(ena_stats)); 980 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 981 if (unlikely(rc)) { 982 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 983 return rc; 984 } 985 986 /* Set of basic statistics from ENA */ 987 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 988 ena_stats.rx_pkts_low); 989 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 990 ena_stats.tx_pkts_low); 991 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 992 ena_stats.rx_bytes_low); 993 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 994 ena_stats.tx_bytes_low); 995 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 996 ena_stats.rx_drops_low); 997 998 /* Driver related stats */ 999 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 1000 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 1001 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 1002 return 0; 1003 } 1004 1005 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1006 { 1007 struct ena_adapter *adapter; 1008 struct ena_com_dev *ena_dev; 1009 int rc = 0; 1010 1011 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1012 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1013 adapter = (struct ena_adapter *)(dev->data->dev_private); 1014 1015 ena_dev = &adapter->ena_dev; 1016 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1017 1018 if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) { 1019 RTE_LOG(ERR, PMD, 1020 "Invalid MTU setting. new_mtu: %d " 1021 "max mtu: %d min mtu: %d\n", 1022 mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU); 1023 return -EINVAL; 1024 } 1025 1026 rc = ena_com_set_dev_mtu(ena_dev, mtu); 1027 if (rc) 1028 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 1029 else 1030 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 1031 1032 return rc; 1033 } 1034 1035 static int ena_start(struct rte_eth_dev *dev) 1036 { 1037 struct ena_adapter *adapter = 1038 (struct ena_adapter *)(dev->data->dev_private); 1039 uint64_t ticks; 1040 int rc = 0; 1041 1042 rc = ena_check_valid_conf(adapter); 1043 if (rc) 1044 return rc; 1045 1046 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 1047 if (rc) 1048 return rc; 1049 1050 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 1051 if (rc) 1052 return rc; 1053 1054 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 1055 ETH_MQ_RX_RSS_FLAG && adapter->rte_dev->data->nb_rx_queues > 0) { 1056 rc = ena_rss_init_default(adapter); 1057 if (rc) 1058 return rc; 1059 } 1060 1061 ena_stats_restart(dev); 1062 1063 adapter->timestamp_wd = rte_get_timer_cycles(); 1064 adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; 1065 1066 ticks = rte_get_timer_hz(); 1067 rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(), 1068 ena_timer_wd_callback, adapter); 1069 1070 adapter->state = ENA_ADAPTER_STATE_RUNNING; 1071 1072 return 0; 1073 } 1074 1075 static void ena_stop(struct rte_eth_dev *dev) 1076 { 1077 struct ena_adapter *adapter = 1078 (struct ena_adapter *)(dev->data->dev_private); 1079 1080 rte_timer_stop_sync(&adapter->timer_wd); 1081 1082 adapter->state = ENA_ADAPTER_STATE_STOPPED; 1083 } 1084 1085 static int ena_queue_restart(struct ena_ring *ring) 1086 { 1087 int rc, bufs_num; 1088 1089 ena_assert_msg(ring->configured == 1, 1090 "Trying to restart unconfigured queue\n"); 1091 1092 ring->next_to_clean = 0; 1093 ring->next_to_use = 0; 1094 1095 if (ring->type == ENA_RING_TYPE_TX) 1096 return 0; 1097 1098 bufs_num = ring->ring_size - 1; 1099 rc = ena_populate_rx_queue(ring, bufs_num); 1100 if (rc != bufs_num) { 1101 PMD_INIT_LOG(ERR, "Failed to populate rx ring !"); 1102 return ENA_COM_FAULT; 1103 } 1104 1105 return 0; 1106 } 1107 1108 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 1109 uint16_t queue_idx, 1110 uint16_t nb_desc, 1111 __rte_unused unsigned int socket_id, 1112 const struct rte_eth_txconf *tx_conf) 1113 { 1114 struct ena_com_create_io_ctx ctx = 1115 /* policy set to _HOST just to satisfy icc compiler */ 1116 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1117 ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 }; 1118 struct ena_ring *txq = NULL; 1119 struct ena_adapter *adapter = 1120 (struct ena_adapter *)(dev->data->dev_private); 1121 unsigned int i; 1122 int ena_qid; 1123 int rc; 1124 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1125 1126 txq = &adapter->tx_ring[queue_idx]; 1127 1128 if (txq->configured) { 1129 RTE_LOG(CRIT, PMD, 1130 "API violation. Queue %d is already configured\n", 1131 queue_idx); 1132 return ENA_COM_FAULT; 1133 } 1134 1135 if (!rte_is_power_of_2(nb_desc)) { 1136 RTE_LOG(ERR, PMD, 1137 "Unsupported size of TX queue: %d is not a power of 2.", 1138 nb_desc); 1139 return -EINVAL; 1140 } 1141 1142 if (nb_desc > adapter->tx_ring_size) { 1143 RTE_LOG(ERR, PMD, 1144 "Unsupported size of TX queue (max size: %d)\n", 1145 adapter->tx_ring_size); 1146 return -EINVAL; 1147 } 1148 1149 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 1150 1151 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1152 ctx.qid = ena_qid; 1153 ctx.msix_vector = -1; /* admin interrupts not used */ 1154 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1155 ctx.queue_size = adapter->tx_ring_size; 1156 ctx.numa_node = ena_cpu_to_node(queue_idx); 1157 1158 rc = ena_com_create_io_queue(ena_dev, &ctx); 1159 if (rc) { 1160 RTE_LOG(ERR, PMD, 1161 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 1162 queue_idx, ena_qid, rc); 1163 return rc; 1164 } 1165 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 1166 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1167 1168 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1169 &txq->ena_com_io_sq, 1170 &txq->ena_com_io_cq); 1171 if (rc) { 1172 RTE_LOG(ERR, PMD, 1173 "Failed to get TX queue handlers. TX queue num %d rc: %d\n", 1174 queue_idx, rc); 1175 goto err_destroy_io_queue; 1176 } 1177 1178 ena_com_update_numa_node(txq->ena_com_io_cq, ctx.numa_node); 1179 1180 txq->port_id = dev->data->port_id; 1181 txq->next_to_clean = 0; 1182 txq->next_to_use = 0; 1183 txq->ring_size = nb_desc; 1184 1185 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1186 sizeof(struct ena_tx_buffer) * 1187 txq->ring_size, 1188 RTE_CACHE_LINE_SIZE); 1189 if (!txq->tx_buffer_info) { 1190 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 1191 rc = -ENOMEM; 1192 goto err_destroy_io_queue; 1193 } 1194 1195 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1196 sizeof(u16) * txq->ring_size, 1197 RTE_CACHE_LINE_SIZE); 1198 if (!txq->empty_tx_reqs) { 1199 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 1200 rc = -ENOMEM; 1201 goto err_free; 1202 } 1203 1204 for (i = 0; i < txq->ring_size; i++) 1205 txq->empty_tx_reqs[i] = i; 1206 1207 if (tx_conf != NULL) { 1208 txq->offloads = 1209 tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1210 } 1211 1212 /* Store pointer to this queue in upper layer */ 1213 txq->configured = 1; 1214 dev->data->tx_queues[queue_idx] = txq; 1215 1216 return 0; 1217 1218 err_free: 1219 rte_free(txq->tx_buffer_info); 1220 1221 err_destroy_io_queue: 1222 ena_com_destroy_io_queue(ena_dev, ena_qid); 1223 return rc; 1224 } 1225 1226 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1227 uint16_t queue_idx, 1228 uint16_t nb_desc, 1229 __rte_unused unsigned int socket_id, 1230 __rte_unused const struct rte_eth_rxconf *rx_conf, 1231 struct rte_mempool *mp) 1232 { 1233 struct ena_com_create_io_ctx ctx = 1234 /* policy set to _HOST just to satisfy icc compiler */ 1235 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1236 ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 }; 1237 struct ena_adapter *adapter = 1238 (struct ena_adapter *)(dev->data->dev_private); 1239 struct ena_ring *rxq = NULL; 1240 uint16_t ena_qid = 0; 1241 int i, rc = 0; 1242 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1243 1244 rxq = &adapter->rx_ring[queue_idx]; 1245 if (rxq->configured) { 1246 RTE_LOG(CRIT, PMD, 1247 "API violation. Queue %d is already configured\n", 1248 queue_idx); 1249 return ENA_COM_FAULT; 1250 } 1251 1252 if (!rte_is_power_of_2(nb_desc)) { 1253 RTE_LOG(ERR, PMD, 1254 "Unsupported size of RX queue: %d is not a power of 2.", 1255 nb_desc); 1256 return -EINVAL; 1257 } 1258 1259 if (nb_desc > adapter->rx_ring_size) { 1260 RTE_LOG(ERR, PMD, 1261 "Unsupported size of RX queue (max size: %d)\n", 1262 adapter->rx_ring_size); 1263 return -EINVAL; 1264 } 1265 1266 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 1267 1268 ctx.qid = ena_qid; 1269 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1270 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1271 ctx.msix_vector = -1; /* admin interrupts not used */ 1272 ctx.queue_size = adapter->rx_ring_size; 1273 ctx.numa_node = ena_cpu_to_node(queue_idx); 1274 1275 rc = ena_com_create_io_queue(ena_dev, &ctx); 1276 if (rc) { 1277 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 1278 queue_idx, rc); 1279 return rc; 1280 } 1281 1282 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 1283 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1284 1285 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1286 &rxq->ena_com_io_sq, 1287 &rxq->ena_com_io_cq); 1288 if (rc) { 1289 RTE_LOG(ERR, PMD, 1290 "Failed to get RX queue handlers. RX queue num %d rc: %d\n", 1291 queue_idx, rc); 1292 ena_com_destroy_io_queue(ena_dev, ena_qid); 1293 return rc; 1294 } 1295 1296 rxq->port_id = dev->data->port_id; 1297 rxq->next_to_clean = 0; 1298 rxq->next_to_use = 0; 1299 rxq->ring_size = nb_desc; 1300 rxq->mb_pool = mp; 1301 1302 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1303 sizeof(struct rte_mbuf *) * nb_desc, 1304 RTE_CACHE_LINE_SIZE); 1305 if (!rxq->rx_buffer_info) { 1306 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 1307 ena_com_destroy_io_queue(ena_dev, ena_qid); 1308 return -ENOMEM; 1309 } 1310 1311 rxq->empty_rx_reqs = rte_zmalloc("rxq->empty_rx_reqs", 1312 sizeof(uint16_t) * nb_desc, 1313 RTE_CACHE_LINE_SIZE); 1314 if (!rxq->empty_rx_reqs) { 1315 RTE_LOG(ERR, PMD, "failed to alloc mem for empty rx reqs\n"); 1316 rte_free(rxq->rx_buffer_info); 1317 rxq->rx_buffer_info = NULL; 1318 ena_com_destroy_io_queue(ena_dev, ena_qid); 1319 return -ENOMEM; 1320 } 1321 1322 for (i = 0; i < nb_desc; i++) 1323 rxq->empty_tx_reqs[i] = i; 1324 1325 /* Store pointer to this queue in upper layer */ 1326 rxq->configured = 1; 1327 dev->data->rx_queues[queue_idx] = rxq; 1328 1329 return rc; 1330 } 1331 1332 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1333 { 1334 unsigned int i; 1335 int rc; 1336 uint16_t ring_size = rxq->ring_size; 1337 uint16_t ring_mask = ring_size - 1; 1338 uint16_t next_to_use = rxq->next_to_use; 1339 uint16_t in_use, req_id; 1340 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 1341 1342 if (unlikely(!count)) 1343 return 0; 1344 1345 in_use = rxq->next_to_use - rxq->next_to_clean; 1346 ena_assert_msg(((in_use + count) < ring_size), "bad ring state"); 1347 1348 count = RTE_MIN(count, 1349 (uint16_t)(ring_size - (next_to_use & ring_mask))); 1350 1351 /* get resources for incoming packets */ 1352 rc = rte_mempool_get_bulk(rxq->mb_pool, 1353 (void **)(&mbufs[next_to_use & ring_mask]), 1354 count); 1355 if (unlikely(rc < 0)) { 1356 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1357 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1358 return 0; 1359 } 1360 1361 for (i = 0; i < count; i++) { 1362 uint16_t next_to_use_masked = next_to_use & ring_mask; 1363 struct rte_mbuf *mbuf = mbufs[next_to_use_masked]; 1364 struct ena_com_buf ebuf; 1365 1366 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 1367 1368 req_id = rxq->empty_rx_reqs[next_to_use_masked]; 1369 rc = validate_rx_req_id(rxq, req_id); 1370 if (unlikely(rc < 0)) 1371 break; 1372 1373 /* prepare physical address for DMA transaction */ 1374 ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; 1375 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1376 /* pass resource to device */ 1377 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 1378 &ebuf, req_id); 1379 if (unlikely(rc)) { 1380 rte_mempool_put_bulk(rxq->mb_pool, (void **)(&mbuf), 1381 count - i); 1382 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 1383 break; 1384 } 1385 next_to_use++; 1386 } 1387 1388 if (unlikely(i < count)) 1389 RTE_LOG(WARNING, PMD, "refilled rx qid %d with only %d " 1390 "buffers (from %d)\n", rxq->id, i, count); 1391 1392 /* When we submitted free recources to device... */ 1393 if (likely(i > 0)) { 1394 /* ...let HW know that it can fill buffers with data 1395 * 1396 * Add memory barrier to make sure the desc were written before 1397 * issue a doorbell 1398 */ 1399 rte_wmb(); 1400 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1401 1402 rxq->next_to_use = next_to_use; 1403 } 1404 1405 return i; 1406 } 1407 1408 static int ena_device_init(struct ena_com_dev *ena_dev, 1409 struct ena_com_dev_get_features_ctx *get_feat_ctx, 1410 bool *wd_state) 1411 { 1412 uint32_t aenq_groups; 1413 int rc; 1414 bool readless_supported; 1415 1416 /* Initialize mmio registers */ 1417 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1418 if (rc) { 1419 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 1420 return rc; 1421 } 1422 1423 /* The PCIe configuration space revision id indicate if mmio reg 1424 * read is disabled. 1425 */ 1426 readless_supported = 1427 !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id 1428 & ENA_MMIO_DISABLE_REG_READ); 1429 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1430 1431 /* reset device */ 1432 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 1433 if (rc) { 1434 RTE_LOG(ERR, PMD, "cannot reset device\n"); 1435 goto err_mmio_read_less; 1436 } 1437 1438 /* check FW version */ 1439 rc = ena_com_validate_version(ena_dev); 1440 if (rc) { 1441 RTE_LOG(ERR, PMD, "device version is too low\n"); 1442 goto err_mmio_read_less; 1443 } 1444 1445 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1446 1447 /* ENA device administration layer init */ 1448 rc = ena_com_admin_init(ena_dev, &aenq_handlers, true); 1449 if (rc) { 1450 RTE_LOG(ERR, PMD, 1451 "cannot initialize ena admin queue with device\n"); 1452 goto err_mmio_read_less; 1453 } 1454 1455 /* To enable the msix interrupts the driver needs to know the number 1456 * of queues. So the driver uses polling mode to retrieve this 1457 * information. 1458 */ 1459 ena_com_set_admin_polling_mode(ena_dev, true); 1460 1461 ena_config_host_info(ena_dev); 1462 1463 /* Get Device Attributes and features */ 1464 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1465 if (rc) { 1466 RTE_LOG(ERR, PMD, 1467 "cannot get attribute for ena device rc= %d\n", rc); 1468 goto err_admin_init; 1469 } 1470 1471 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 1472 BIT(ENA_ADMIN_NOTIFICATION) | 1473 BIT(ENA_ADMIN_KEEP_ALIVE) | 1474 BIT(ENA_ADMIN_FATAL_ERROR) | 1475 BIT(ENA_ADMIN_WARNING); 1476 1477 aenq_groups &= get_feat_ctx->aenq.supported_groups; 1478 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 1479 if (rc) { 1480 RTE_LOG(ERR, PMD, "Cannot configure aenq groups rc: %d\n", rc); 1481 goto err_admin_init; 1482 } 1483 1484 *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 1485 1486 return 0; 1487 1488 err_admin_init: 1489 ena_com_admin_destroy(ena_dev); 1490 1491 err_mmio_read_less: 1492 ena_com_mmio_reg_read_request_destroy(ena_dev); 1493 1494 return rc; 1495 } 1496 1497 static void ena_interrupt_handler_rte(void *cb_arg) 1498 { 1499 struct ena_adapter *adapter = (struct ena_adapter *)cb_arg; 1500 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1501 1502 ena_com_admin_q_comp_intr_handler(ena_dev); 1503 if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED)) 1504 ena_com_aenq_intr_handler(ena_dev, adapter); 1505 } 1506 1507 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 1508 { 1509 if (!adapter->wd_state) 1510 return; 1511 1512 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 1513 return; 1514 1515 if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >= 1516 adapter->keep_alive_timeout)) { 1517 RTE_LOG(ERR, PMD, "Keep alive timeout\n"); 1518 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; 1519 adapter->trigger_reset = true; 1520 } 1521 } 1522 1523 /* Check if admin queue is enabled */ 1524 static void check_for_admin_com_state(struct ena_adapter *adapter) 1525 { 1526 if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) { 1527 RTE_LOG(ERR, PMD, "ENA admin queue is not in running state!\n"); 1528 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; 1529 adapter->trigger_reset = true; 1530 } 1531 } 1532 1533 static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer, 1534 void *arg) 1535 { 1536 struct ena_adapter *adapter = (struct ena_adapter *)arg; 1537 struct rte_eth_dev *dev = adapter->rte_dev; 1538 1539 check_for_missing_keep_alive(adapter); 1540 check_for_admin_com_state(adapter); 1541 1542 if (unlikely(adapter->trigger_reset)) { 1543 RTE_LOG(ERR, PMD, "Trigger reset is on\n"); 1544 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 1545 NULL); 1546 } 1547 } 1548 1549 static int ena_calc_io_queue_num(__rte_unused struct ena_com_dev *ena_dev, 1550 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1551 { 1552 int io_sq_num, io_cq_num, io_queue_num; 1553 1554 io_sq_num = get_feat_ctx->max_queues.max_sq_num; 1555 io_cq_num = get_feat_ctx->max_queues.max_cq_num; 1556 1557 io_queue_num = RTE_MIN(io_sq_num, io_cq_num); 1558 1559 if (unlikely(io_queue_num == 0)) { 1560 RTE_LOG(ERR, PMD, "Number of IO queues should not be 0\n"); 1561 return -EFAULT; 1562 } 1563 1564 return io_queue_num; 1565 } 1566 1567 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1568 { 1569 struct rte_pci_device *pci_dev; 1570 struct rte_intr_handle *intr_handle; 1571 struct ena_adapter *adapter = 1572 (struct ena_adapter *)(eth_dev->data->dev_private); 1573 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1574 struct ena_com_dev_get_features_ctx get_feat_ctx; 1575 int queue_size, rc; 1576 u16 tx_sgl_size = 0; 1577 1578 static int adapters_found; 1579 bool wd_state; 1580 1581 memset(adapter, 0, sizeof(struct ena_adapter)); 1582 ena_dev = &adapter->ena_dev; 1583 1584 eth_dev->dev_ops = &ena_dev_ops; 1585 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1586 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1587 eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; 1588 adapter->rte_eth_dev_data = eth_dev->data; 1589 adapter->rte_dev = eth_dev; 1590 1591 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1592 return 0; 1593 1594 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1595 adapter->pdev = pci_dev; 1596 1597 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", 1598 pci_dev->addr.domain, 1599 pci_dev->addr.bus, 1600 pci_dev->addr.devid, 1601 pci_dev->addr.function); 1602 1603 intr_handle = &pci_dev->intr_handle; 1604 1605 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1606 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1607 1608 if (!adapter->regs) { 1609 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)", 1610 ENA_REGS_BAR); 1611 return -ENXIO; 1612 } 1613 1614 ena_dev->reg_bar = adapter->regs; 1615 ena_dev->dmadev = adapter->pdev; 1616 1617 adapter->id_number = adapters_found; 1618 1619 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1620 adapter->id_number); 1621 1622 /* device specific initialization routine */ 1623 rc = ena_device_init(ena_dev, &get_feat_ctx, &wd_state); 1624 if (rc) { 1625 PMD_INIT_LOG(CRIT, "Failed to init ENA device"); 1626 goto err; 1627 } 1628 adapter->wd_state = wd_state; 1629 1630 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1631 adapter->num_queues = ena_calc_io_queue_num(ena_dev, 1632 &get_feat_ctx); 1633 1634 queue_size = ena_calc_queue_size(ena_dev, &tx_sgl_size, &get_feat_ctx); 1635 if (queue_size <= 0 || adapter->num_queues <= 0) { 1636 rc = -EFAULT; 1637 goto err_device_destroy; 1638 } 1639 1640 adapter->tx_ring_size = queue_size; 1641 adapter->rx_ring_size = queue_size; 1642 1643 adapter->max_tx_sgl_size = tx_sgl_size; 1644 1645 /* prepare ring structures */ 1646 ena_init_rings(adapter); 1647 1648 ena_config_debug_area(adapter); 1649 1650 /* Set max MTU for this device */ 1651 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1652 1653 /* set device support for TSO */ 1654 adapter->tso4_supported = get_feat_ctx.offload.tx & 1655 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; 1656 1657 /* Copy MAC address and point DPDK to it */ 1658 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1659 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1660 (struct ether_addr *)adapter->mac_addr); 1661 1662 adapter->drv_stats = rte_zmalloc("adapter stats", 1663 sizeof(*adapter->drv_stats), 1664 RTE_CACHE_LINE_SIZE); 1665 if (!adapter->drv_stats) { 1666 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1667 rc = -ENOMEM; 1668 goto err_delete_debug_area; 1669 } 1670 1671 rte_intr_callback_register(intr_handle, 1672 ena_interrupt_handler_rte, 1673 adapter); 1674 rte_intr_enable(intr_handle); 1675 ena_com_set_admin_polling_mode(ena_dev, false); 1676 ena_com_admin_aenq_enable(ena_dev); 1677 1678 if (adapters_found == 0) 1679 rte_timer_subsystem_init(); 1680 rte_timer_init(&adapter->timer_wd); 1681 1682 adapters_found++; 1683 adapter->state = ENA_ADAPTER_STATE_INIT; 1684 1685 return 0; 1686 1687 err_delete_debug_area: 1688 ena_com_delete_debug_area(ena_dev); 1689 1690 err_device_destroy: 1691 ena_com_delete_host_info(ena_dev); 1692 ena_com_admin_destroy(ena_dev); 1693 1694 err: 1695 return rc; 1696 } 1697 1698 static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev) 1699 { 1700 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1701 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1702 struct ena_adapter *adapter = 1703 (struct ena_adapter *)(eth_dev->data->dev_private); 1704 1705 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1706 return -EPERM; 1707 1708 if (adapter->state != ENA_ADAPTER_STATE_CLOSED) 1709 ena_close(eth_dev); 1710 1711 eth_dev->dev_ops = NULL; 1712 eth_dev->rx_pkt_burst = NULL; 1713 eth_dev->tx_pkt_burst = NULL; 1714 eth_dev->tx_pkt_prepare = NULL; 1715 1716 rte_free(adapter->drv_stats); 1717 adapter->drv_stats = NULL; 1718 1719 rte_intr_disable(intr_handle); 1720 rte_intr_callback_unregister(intr_handle, 1721 ena_interrupt_handler_rte, 1722 adapter); 1723 1724 adapter->state = ENA_ADAPTER_STATE_FREE; 1725 1726 return 0; 1727 } 1728 1729 static int ena_dev_configure(struct rte_eth_dev *dev) 1730 { 1731 struct ena_adapter *adapter = 1732 (struct ena_adapter *)(dev->data->dev_private); 1733 1734 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1735 1736 adapter->tx_selected_offloads = dev->data->dev_conf.txmode.offloads; 1737 adapter->rx_selected_offloads = dev->data->dev_conf.rxmode.offloads; 1738 return 0; 1739 } 1740 1741 static void ena_init_rings(struct ena_adapter *adapter) 1742 { 1743 int i; 1744 1745 for (i = 0; i < adapter->num_queues; i++) { 1746 struct ena_ring *ring = &adapter->tx_ring[i]; 1747 1748 ring->configured = 0; 1749 ring->type = ENA_RING_TYPE_TX; 1750 ring->adapter = adapter; 1751 ring->id = i; 1752 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1753 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1754 ring->sgl_size = adapter->max_tx_sgl_size; 1755 } 1756 1757 for (i = 0; i < adapter->num_queues; i++) { 1758 struct ena_ring *ring = &adapter->rx_ring[i]; 1759 1760 ring->configured = 0; 1761 ring->type = ENA_RING_TYPE_RX; 1762 ring->adapter = adapter; 1763 ring->id = i; 1764 } 1765 } 1766 1767 static void ena_infos_get(struct rte_eth_dev *dev, 1768 struct rte_eth_dev_info *dev_info) 1769 { 1770 struct ena_adapter *adapter; 1771 struct ena_com_dev *ena_dev; 1772 struct ena_com_dev_get_features_ctx feat; 1773 uint64_t rx_feat = 0, tx_feat = 0; 1774 int rc = 0; 1775 1776 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1777 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1778 adapter = (struct ena_adapter *)(dev->data->dev_private); 1779 1780 ena_dev = &adapter->ena_dev; 1781 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1782 1783 dev_info->speed_capa = 1784 ETH_LINK_SPEED_1G | 1785 ETH_LINK_SPEED_2_5G | 1786 ETH_LINK_SPEED_5G | 1787 ETH_LINK_SPEED_10G | 1788 ETH_LINK_SPEED_25G | 1789 ETH_LINK_SPEED_40G | 1790 ETH_LINK_SPEED_50G | 1791 ETH_LINK_SPEED_100G; 1792 1793 /* Get supported features from HW */ 1794 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1795 if (unlikely(rc)) { 1796 RTE_LOG(ERR, PMD, 1797 "Cannot get attribute for ena device rc= %d\n", rc); 1798 return; 1799 } 1800 1801 /* Set Tx & Rx features available for device */ 1802 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1803 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1804 1805 if (feat.offload.tx & 1806 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1807 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1808 DEV_TX_OFFLOAD_UDP_CKSUM | 1809 DEV_TX_OFFLOAD_TCP_CKSUM; 1810 1811 if (feat.offload.rx_supported & 1812 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1813 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1814 DEV_RX_OFFLOAD_UDP_CKSUM | 1815 DEV_RX_OFFLOAD_TCP_CKSUM; 1816 1817 rx_feat |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1818 1819 /* Inform framework about available features */ 1820 dev_info->rx_offload_capa = rx_feat; 1821 dev_info->rx_queue_offload_capa = rx_feat; 1822 dev_info->tx_offload_capa = tx_feat; 1823 dev_info->tx_queue_offload_capa = tx_feat; 1824 1825 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1826 dev_info->max_rx_pktlen = adapter->max_mtu; 1827 dev_info->max_mac_addrs = 1; 1828 1829 dev_info->max_rx_queues = adapter->num_queues; 1830 dev_info->max_tx_queues = adapter->num_queues; 1831 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1832 1833 adapter->tx_supported_offloads = tx_feat; 1834 adapter->rx_supported_offloads = rx_feat; 1835 1836 dev_info->rx_desc_lim.nb_max = ENA_MAX_RING_DESC; 1837 dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC; 1838 1839 dev_info->tx_desc_lim.nb_max = ENA_MAX_RING_DESC; 1840 dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC; 1841 dev_info->tx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1842 feat.max_queues.max_packet_tx_descs); 1843 dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1844 feat.max_queues.max_packet_tx_descs); 1845 } 1846 1847 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1848 uint16_t nb_pkts) 1849 { 1850 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1851 unsigned int ring_size = rx_ring->ring_size; 1852 unsigned int ring_mask = ring_size - 1; 1853 uint16_t next_to_clean = rx_ring->next_to_clean; 1854 uint16_t desc_in_use = 0; 1855 uint16_t req_id; 1856 unsigned int recv_idx = 0; 1857 struct rte_mbuf *mbuf = NULL; 1858 struct rte_mbuf *mbuf_head = NULL; 1859 struct rte_mbuf *mbuf_prev = NULL; 1860 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1861 unsigned int completed; 1862 1863 struct ena_com_rx_ctx ena_rx_ctx; 1864 int rc = 0; 1865 1866 /* Check adapter state */ 1867 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1868 RTE_LOG(ALERT, PMD, 1869 "Trying to receive pkts while device is NOT running\n"); 1870 return 0; 1871 } 1872 1873 desc_in_use = rx_ring->next_to_use - next_to_clean; 1874 if (unlikely(nb_pkts > desc_in_use)) 1875 nb_pkts = desc_in_use; 1876 1877 for (completed = 0; completed < nb_pkts; completed++) { 1878 int segments = 0; 1879 1880 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1881 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1882 ena_rx_ctx.descs = 0; 1883 /* receive packet context */ 1884 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1885 rx_ring->ena_com_io_sq, 1886 &ena_rx_ctx); 1887 if (unlikely(rc)) { 1888 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1889 rx_ring->adapter->trigger_reset = true; 1890 return 0; 1891 } 1892 1893 if (unlikely(ena_rx_ctx.descs == 0)) 1894 break; 1895 1896 while (segments < ena_rx_ctx.descs) { 1897 req_id = ena_rx_ctx.ena_bufs[segments].req_id; 1898 rc = validate_rx_req_id(rx_ring, req_id); 1899 if (unlikely(rc)) 1900 break; 1901 1902 mbuf = rx_buff_info[req_id]; 1903 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1904 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1905 mbuf->refcnt = 1; 1906 mbuf->next = NULL; 1907 if (unlikely(segments == 0)) { 1908 mbuf->nb_segs = ena_rx_ctx.descs; 1909 mbuf->port = rx_ring->port_id; 1910 mbuf->pkt_len = 0; 1911 mbuf_head = mbuf; 1912 } else { 1913 /* for multi-segment pkts create mbuf chain */ 1914 mbuf_prev->next = mbuf; 1915 } 1916 mbuf_head->pkt_len += mbuf->data_len; 1917 1918 mbuf_prev = mbuf; 1919 rx_ring->empty_rx_reqs[next_to_clean & ring_mask] = 1920 req_id; 1921 segments++; 1922 next_to_clean++; 1923 } 1924 1925 /* fill mbuf attributes if any */ 1926 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1927 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1928 1929 /* pass to DPDK application head mbuf */ 1930 rx_pkts[recv_idx] = mbuf_head; 1931 recv_idx++; 1932 } 1933 1934 rx_ring->next_to_clean = next_to_clean; 1935 1936 desc_in_use = desc_in_use - completed + 1; 1937 /* Burst refill to save doorbells, memory barriers, const interval */ 1938 if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) 1939 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use); 1940 1941 return recv_idx; 1942 } 1943 1944 static uint16_t 1945 eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1946 uint16_t nb_pkts) 1947 { 1948 int32_t ret; 1949 uint32_t i; 1950 struct rte_mbuf *m; 1951 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1952 struct ipv4_hdr *ip_hdr; 1953 uint64_t ol_flags; 1954 uint16_t frag_field; 1955 1956 for (i = 0; i != nb_pkts; i++) { 1957 m = tx_pkts[i]; 1958 ol_flags = m->ol_flags; 1959 1960 if (!(ol_flags & PKT_TX_IPV4)) 1961 continue; 1962 1963 /* If there was not L2 header length specified, assume it is 1964 * length of the ethernet header. 1965 */ 1966 if (unlikely(m->l2_len == 0)) 1967 m->l2_len = sizeof(struct ether_hdr); 1968 1969 ip_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, 1970 m->l2_len); 1971 frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); 1972 1973 if ((frag_field & IPV4_HDR_DF_FLAG) != 0) { 1974 m->packet_type |= RTE_PTYPE_L4_NONFRAG; 1975 1976 /* If IPv4 header has DF flag enabled and TSO support is 1977 * disabled, partial chcecksum should not be calculated. 1978 */ 1979 if (!tx_ring->adapter->tso4_supported) 1980 continue; 1981 } 1982 1983 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || 1984 (ol_flags & PKT_TX_L4_MASK) == 1985 PKT_TX_SCTP_CKSUM) { 1986 rte_errno = -ENOTSUP; 1987 return i; 1988 } 1989 1990 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1991 ret = rte_validate_tx_offload(m); 1992 if (ret != 0) { 1993 rte_errno = ret; 1994 return i; 1995 } 1996 #endif 1997 1998 /* In case we are supposed to TSO and have DF not set (DF=0) 1999 * hardware must be provided with partial checksum, otherwise 2000 * it will take care of necessary calculations. 2001 */ 2002 2003 ret = rte_net_intel_cksum_flags_prepare(m, 2004 ol_flags & ~PKT_TX_TCP_SEG); 2005 if (ret != 0) { 2006 rte_errno = ret; 2007 return i; 2008 } 2009 } 2010 2011 return i; 2012 } 2013 2014 static void ena_update_hints(struct ena_adapter *adapter, 2015 struct ena_admin_ena_hw_hints *hints) 2016 { 2017 if (hints->admin_completion_tx_timeout) 2018 adapter->ena_dev.admin_queue.completion_timeout = 2019 hints->admin_completion_tx_timeout * 1000; 2020 2021 if (hints->mmio_read_timeout) 2022 /* convert to usec */ 2023 adapter->ena_dev.mmio_read.reg_read_to = 2024 hints->mmio_read_timeout * 1000; 2025 2026 if (hints->driver_watchdog_timeout) { 2027 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2028 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 2029 else 2030 // Convert msecs to ticks 2031 adapter->keep_alive_timeout = 2032 (hints->driver_watchdog_timeout * 2033 rte_get_timer_hz()) / 1000; 2034 } 2035 } 2036 2037 static int ena_check_and_linearize_mbuf(struct ena_ring *tx_ring, 2038 struct rte_mbuf *mbuf) 2039 { 2040 int num_segments, rc; 2041 2042 num_segments = mbuf->nb_segs; 2043 2044 if (likely(num_segments < tx_ring->sgl_size)) 2045 return 0; 2046 2047 rc = rte_pktmbuf_linearize(mbuf); 2048 if (unlikely(rc)) 2049 RTE_LOG(WARNING, PMD, "Mbuf linearize failed\n"); 2050 2051 return rc; 2052 } 2053 2054 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2055 uint16_t nb_pkts) 2056 { 2057 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2058 uint16_t next_to_use = tx_ring->next_to_use; 2059 uint16_t next_to_clean = tx_ring->next_to_clean; 2060 struct rte_mbuf *mbuf; 2061 unsigned int ring_size = tx_ring->ring_size; 2062 unsigned int ring_mask = ring_size - 1; 2063 struct ena_com_tx_ctx ena_tx_ctx; 2064 struct ena_tx_buffer *tx_info; 2065 struct ena_com_buf *ebuf; 2066 uint16_t rc, req_id, total_tx_descs = 0; 2067 uint16_t sent_idx = 0, empty_tx_reqs; 2068 int nb_hw_desc; 2069 2070 /* Check adapter state */ 2071 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2072 RTE_LOG(ALERT, PMD, 2073 "Trying to xmit pkts while device is NOT running\n"); 2074 return 0; 2075 } 2076 2077 empty_tx_reqs = ring_size - (next_to_use - next_to_clean); 2078 if (nb_pkts > empty_tx_reqs) 2079 nb_pkts = empty_tx_reqs; 2080 2081 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 2082 mbuf = tx_pkts[sent_idx]; 2083 2084 rc = ena_check_and_linearize_mbuf(tx_ring, mbuf); 2085 if (unlikely(rc)) 2086 break; 2087 2088 req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; 2089 tx_info = &tx_ring->tx_buffer_info[req_id]; 2090 tx_info->mbuf = mbuf; 2091 tx_info->num_of_bufs = 0; 2092 ebuf = tx_info->bufs; 2093 2094 /* Prepare TX context */ 2095 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 2096 memset(&ena_tx_ctx.ena_meta, 0x0, 2097 sizeof(struct ena_com_tx_meta)); 2098 ena_tx_ctx.ena_bufs = ebuf; 2099 ena_tx_ctx.req_id = req_id; 2100 if (tx_ring->tx_mem_queue_type == 2101 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2102 /* prepare the push buffer with 2103 * virtual address of the data 2104 */ 2105 ena_tx_ctx.header_len = 2106 RTE_MIN(mbuf->data_len, 2107 tx_ring->tx_max_header_size); 2108 ena_tx_ctx.push_header = 2109 (void *)((char *)mbuf->buf_addr + 2110 mbuf->data_off); 2111 } /* there's no else as we take advantage of memset zeroing */ 2112 2113 /* Set TX offloads flags, if applicable */ 2114 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads); 2115 2116 if (unlikely(mbuf->ol_flags & 2117 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 2118 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 2119 2120 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 2121 2122 /* Process first segment taking into 2123 * consideration pushed header 2124 */ 2125 if (mbuf->data_len > ena_tx_ctx.header_len) { 2126 ebuf->paddr = mbuf->buf_iova + 2127 mbuf->data_off + 2128 ena_tx_ctx.header_len; 2129 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 2130 ebuf++; 2131 tx_info->num_of_bufs++; 2132 } 2133 2134 while ((mbuf = mbuf->next) != NULL) { 2135 ebuf->paddr = mbuf->buf_iova + mbuf->data_off; 2136 ebuf->len = mbuf->data_len; 2137 ebuf++; 2138 tx_info->num_of_bufs++; 2139 } 2140 2141 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 2142 2143 /* Write data to device */ 2144 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 2145 &ena_tx_ctx, &nb_hw_desc); 2146 if (unlikely(rc)) 2147 break; 2148 2149 tx_info->tx_descs = nb_hw_desc; 2150 2151 next_to_use++; 2152 } 2153 2154 /* If there are ready packets to be xmitted... */ 2155 if (sent_idx > 0) { 2156 /* ...let HW do its best :-) */ 2157 rte_wmb(); 2158 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2159 2160 tx_ring->next_to_use = next_to_use; 2161 } 2162 2163 /* Clear complete packets */ 2164 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 2165 rc = validate_tx_req_id(tx_ring, req_id); 2166 if (rc) 2167 break; 2168 2169 /* Get Tx info & store how many descs were processed */ 2170 tx_info = &tx_ring->tx_buffer_info[req_id]; 2171 total_tx_descs += tx_info->tx_descs; 2172 2173 /* Free whole mbuf chain */ 2174 mbuf = tx_info->mbuf; 2175 rte_pktmbuf_free(mbuf); 2176 tx_info->mbuf = NULL; 2177 2178 /* Put back descriptor to the ring for reuse */ 2179 tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; 2180 next_to_clean++; 2181 2182 /* If too many descs to clean, leave it for another run */ 2183 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 2184 break; 2185 } 2186 2187 if (total_tx_descs > 0) { 2188 /* acknowledge completion of sent packets */ 2189 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 2190 tx_ring->next_to_clean = next_to_clean; 2191 } 2192 2193 return sent_idx; 2194 } 2195 2196 /********************************************************************* 2197 * PMD configuration 2198 *********************************************************************/ 2199 static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2200 struct rte_pci_device *pci_dev) 2201 { 2202 return rte_eth_dev_pci_generic_probe(pci_dev, 2203 sizeof(struct ena_adapter), eth_ena_dev_init); 2204 } 2205 2206 static int eth_ena_pci_remove(struct rte_pci_device *pci_dev) 2207 { 2208 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ena_dev_uninit); 2209 } 2210 2211 static struct rte_pci_driver rte_ena_pmd = { 2212 .id_table = pci_id_ena_map, 2213 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 2214 RTE_PCI_DRV_WC_ACTIVATE, 2215 .probe = eth_ena_pci_probe, 2216 .remove = eth_ena_pci_remove, 2217 }; 2218 2219 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd); 2220 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 2221 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci"); 2222 2223 RTE_INIT(ena_init_log) 2224 { 2225 ena_logtype_init = rte_log_register("pmd.net.ena.init"); 2226 if (ena_logtype_init >= 0) 2227 rte_log_set_level(ena_logtype_init, RTE_LOG_NOTICE); 2228 ena_logtype_driver = rte_log_register("pmd.net.ena.driver"); 2229 if (ena_logtype_driver >= 0) 2230 rte_log_set_level(ena_logtype_driver, RTE_LOG_NOTICE); 2231 } 2232 2233 /****************************************************************************** 2234 ******************************** AENQ Handlers ******************************* 2235 *****************************************************************************/ 2236 static void ena_update_on_link_change(void *adapter_data, 2237 struct ena_admin_aenq_entry *aenq_e) 2238 { 2239 struct rte_eth_dev *eth_dev; 2240 struct ena_adapter *adapter; 2241 struct ena_admin_aenq_link_change_desc *aenq_link_desc; 2242 uint32_t status; 2243 2244 adapter = (struct ena_adapter *)adapter_data; 2245 aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 2246 eth_dev = adapter->rte_dev; 2247 2248 status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc); 2249 adapter->link_status = status; 2250 2251 ena_link_update(eth_dev, 0); 2252 _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2253 } 2254 2255 static void ena_notification(void *data, 2256 struct ena_admin_aenq_entry *aenq_e) 2257 { 2258 struct ena_adapter *adapter = (struct ena_adapter *)data; 2259 struct ena_admin_ena_hw_hints *hints; 2260 2261 if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION) 2262 RTE_LOG(WARNING, PMD, "Invalid group(%x) expected %x\n", 2263 aenq_e->aenq_common_desc.group, 2264 ENA_ADMIN_NOTIFICATION); 2265 2266 switch (aenq_e->aenq_common_desc.syndrom) { 2267 case ENA_ADMIN_UPDATE_HINTS: 2268 hints = (struct ena_admin_ena_hw_hints *) 2269 (&aenq_e->inline_data_w4); 2270 ena_update_hints(adapter, hints); 2271 break; 2272 default: 2273 RTE_LOG(ERR, PMD, "Invalid aenq notification link state %d\n", 2274 aenq_e->aenq_common_desc.syndrom); 2275 } 2276 } 2277 2278 static void ena_keep_alive(void *adapter_data, 2279 __rte_unused struct ena_admin_aenq_entry *aenq_e) 2280 { 2281 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 2282 2283 adapter->timestamp_wd = rte_get_timer_cycles(); 2284 } 2285 2286 /** 2287 * This handler will called for unknown event group or unimplemented handlers 2288 **/ 2289 static void unimplemented_aenq_handler(__rte_unused void *data, 2290 __rte_unused struct ena_admin_aenq_entry *aenq_e) 2291 { 2292 RTE_LOG(ERR, PMD, "Unknown event was received or event with " 2293 "unimplemented handler\n"); 2294 } 2295 2296 static struct ena_aenq_handlers aenq_handlers = { 2297 .handlers = { 2298 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 2299 [ENA_ADMIN_NOTIFICATION] = ena_notification, 2300 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive 2301 }, 2302 .unimplemented_handler = unimplemented_aenq_handler 2303 }; 2304