1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #include <rte_string_fns.h> 7 #include <rte_ether.h> 8 #include <ethdev_driver.h> 9 #include <ethdev_pci.h> 10 #include <rte_tcp.h> 11 #include <rte_atomic.h> 12 #include <rte_dev.h> 13 #include <rte_errno.h> 14 #include <rte_version.h> 15 #include <rte_net.h> 16 #include <rte_kvargs.h> 17 18 #include "ena_ethdev.h" 19 #include "ena_logs.h" 20 #include "ena_platform.h" 21 #include "ena_com.h" 22 #include "ena_eth_com.h" 23 24 #include <ena_common_defs.h> 25 #include <ena_regs_defs.h> 26 #include <ena_admin_defs.h> 27 #include <ena_eth_io_defs.h> 28 29 #define DRV_MODULE_VER_MAJOR 2 30 #define DRV_MODULE_VER_MINOR 3 31 #define DRV_MODULE_VER_SUBMINOR 0 32 33 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 34 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 35 /*reverse version of ENA_IO_RXQ_IDX*/ 36 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 37 38 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 39 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 40 41 #define GET_L4_HDR_LEN(mbuf) \ 42 ((rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, \ 43 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 44 45 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 46 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 47 #define ENA_HASH_KEY_SIZE 40 48 #define ETH_GSTRING_LEN 32 49 50 #define ARRAY_SIZE(x) RTE_DIM(x) 51 52 #define ENA_MIN_RING_DESC 128 53 54 #define ENA_PTYPE_HAS_HASH (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP) 55 56 enum ethtool_stringset { 57 ETH_SS_TEST = 0, 58 ETH_SS_STATS, 59 }; 60 61 struct ena_stats { 62 char name[ETH_GSTRING_LEN]; 63 int stat_offset; 64 }; 65 66 #define ENA_STAT_ENTRY(stat, stat_type) { \ 67 .name = #stat, \ 68 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 69 } 70 71 #define ENA_STAT_RX_ENTRY(stat) \ 72 ENA_STAT_ENTRY(stat, rx) 73 74 #define ENA_STAT_TX_ENTRY(stat) \ 75 ENA_STAT_ENTRY(stat, tx) 76 77 #define ENA_STAT_ENI_ENTRY(stat) \ 78 ENA_STAT_ENTRY(stat, eni) 79 80 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 81 ENA_STAT_ENTRY(stat, dev) 82 83 /* Device arguments */ 84 #define ENA_DEVARG_LARGE_LLQ_HDR "large_llq_hdr" 85 86 /* 87 * Each rte_memzone should have unique name. 88 * To satisfy it, count number of allocation and add it to name. 89 */ 90 rte_atomic64_t ena_alloc_cnt; 91 92 static const struct ena_stats ena_stats_global_strings[] = { 93 ENA_STAT_GLOBAL_ENTRY(wd_expired), 94 ENA_STAT_GLOBAL_ENTRY(dev_start), 95 ENA_STAT_GLOBAL_ENTRY(dev_stop), 96 ENA_STAT_GLOBAL_ENTRY(tx_drops), 97 }; 98 99 static const struct ena_stats ena_stats_eni_strings[] = { 100 ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded), 101 ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded), 102 ENA_STAT_ENI_ENTRY(pps_allowance_exceeded), 103 ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded), 104 ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded), 105 }; 106 107 static const struct ena_stats ena_stats_tx_strings[] = { 108 ENA_STAT_TX_ENTRY(cnt), 109 ENA_STAT_TX_ENTRY(bytes), 110 ENA_STAT_TX_ENTRY(prepare_ctx_err), 111 ENA_STAT_TX_ENTRY(linearize), 112 ENA_STAT_TX_ENTRY(linearize_failed), 113 ENA_STAT_TX_ENTRY(tx_poll), 114 ENA_STAT_TX_ENTRY(doorbells), 115 ENA_STAT_TX_ENTRY(bad_req_id), 116 ENA_STAT_TX_ENTRY(available_desc), 117 }; 118 119 static const struct ena_stats ena_stats_rx_strings[] = { 120 ENA_STAT_RX_ENTRY(cnt), 121 ENA_STAT_RX_ENTRY(bytes), 122 ENA_STAT_RX_ENTRY(refill_partial), 123 ENA_STAT_RX_ENTRY(bad_csum), 124 ENA_STAT_RX_ENTRY(mbuf_alloc_fail), 125 ENA_STAT_RX_ENTRY(bad_desc_num), 126 ENA_STAT_RX_ENTRY(bad_req_id), 127 }; 128 129 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 130 #define ENA_STATS_ARRAY_ENI ARRAY_SIZE(ena_stats_eni_strings) 131 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 132 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 133 134 #define QUEUE_OFFLOADS (DEV_TX_OFFLOAD_TCP_CKSUM |\ 135 DEV_TX_OFFLOAD_UDP_CKSUM |\ 136 DEV_TX_OFFLOAD_IPV4_CKSUM |\ 137 DEV_TX_OFFLOAD_TCP_TSO) 138 #define MBUF_OFFLOADS (PKT_TX_L4_MASK |\ 139 PKT_TX_IP_CKSUM |\ 140 PKT_TX_TCP_SEG) 141 142 /** Vendor ID used by Amazon devices */ 143 #define PCI_VENDOR_ID_AMAZON 0x1D0F 144 /** Amazon devices */ 145 #define PCI_DEVICE_ID_ENA_VF 0xEC20 146 #define PCI_DEVICE_ID_ENA_VF_RSERV0 0xEC21 147 148 #define ENA_TX_OFFLOAD_MASK (\ 149 PKT_TX_L4_MASK | \ 150 PKT_TX_IPV6 | \ 151 PKT_TX_IPV4 | \ 152 PKT_TX_IP_CKSUM | \ 153 PKT_TX_TCP_SEG) 154 155 #define ENA_TX_OFFLOAD_NOTSUP_MASK \ 156 (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) 157 158 static const struct rte_pci_id pci_id_ena_map[] = { 159 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 160 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF_RSERV0) }, 161 { .device_id = 0 }, 162 }; 163 164 static struct ena_aenq_handlers aenq_handlers; 165 166 static int ena_device_init(struct ena_com_dev *ena_dev, 167 struct rte_pci_device *pdev, 168 struct ena_com_dev_get_features_ctx *get_feat_ctx, 169 bool *wd_state); 170 static int ena_dev_configure(struct rte_eth_dev *dev); 171 static void ena_tx_map_mbuf(struct ena_ring *tx_ring, 172 struct ena_tx_buffer *tx_info, 173 struct rte_mbuf *mbuf, 174 void **push_header, 175 uint16_t *header_len); 176 static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf); 177 static void ena_tx_cleanup(struct ena_ring *tx_ring); 178 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 179 uint16_t nb_pkts); 180 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 181 uint16_t nb_pkts); 182 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 183 uint16_t nb_desc, unsigned int socket_id, 184 const struct rte_eth_txconf *tx_conf); 185 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 186 uint16_t nb_desc, unsigned int socket_id, 187 const struct rte_eth_rxconf *rx_conf, 188 struct rte_mempool *mp); 189 static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len); 190 static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, 191 struct ena_com_rx_buf_info *ena_bufs, 192 uint32_t descs, 193 uint16_t *next_to_clean, 194 uint8_t offset); 195 static uint16_t eth_ena_recv_pkts(void *rx_queue, 196 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 197 static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, 198 struct rte_mbuf *mbuf, uint16_t id); 199 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 200 static void ena_init_rings(struct ena_adapter *adapter, 201 bool disable_meta_caching); 202 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 203 static int ena_start(struct rte_eth_dev *dev); 204 static int ena_stop(struct rte_eth_dev *dev); 205 static int ena_close(struct rte_eth_dev *dev); 206 static int ena_dev_reset(struct rte_eth_dev *dev); 207 static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 208 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 209 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 210 static void ena_rx_queue_release(void *queue); 211 static void ena_tx_queue_release(void *queue); 212 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 213 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 214 static int ena_link_update(struct rte_eth_dev *dev, 215 int wait_to_complete); 216 static int ena_create_io_queue(struct ena_ring *ring); 217 static void ena_queue_stop(struct ena_ring *ring); 218 static void ena_queue_stop_all(struct rte_eth_dev *dev, 219 enum ena_ring_type ring_type); 220 static int ena_queue_start(struct ena_ring *ring); 221 static int ena_queue_start_all(struct rte_eth_dev *dev, 222 enum ena_ring_type ring_type); 223 static void ena_stats_restart(struct rte_eth_dev *dev); 224 static int ena_infos_get(struct rte_eth_dev *dev, 225 struct rte_eth_dev_info *dev_info); 226 static int ena_rss_reta_update(struct rte_eth_dev *dev, 227 struct rte_eth_rss_reta_entry64 *reta_conf, 228 uint16_t reta_size); 229 static int ena_rss_reta_query(struct rte_eth_dev *dev, 230 struct rte_eth_rss_reta_entry64 *reta_conf, 231 uint16_t reta_size); 232 static void ena_interrupt_handler_rte(void *cb_arg); 233 static void ena_timer_wd_callback(struct rte_timer *timer, void *arg); 234 static void ena_destroy_device(struct rte_eth_dev *eth_dev); 235 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev); 236 static int ena_xstats_get_names(struct rte_eth_dev *dev, 237 struct rte_eth_xstat_name *xstats_names, 238 unsigned int n); 239 static int ena_xstats_get(struct rte_eth_dev *dev, 240 struct rte_eth_xstat *stats, 241 unsigned int n); 242 static int ena_xstats_get_by_id(struct rte_eth_dev *dev, 243 const uint64_t *ids, 244 uint64_t *values, 245 unsigned int n); 246 static int ena_process_bool_devarg(const char *key, 247 const char *value, 248 void *opaque); 249 static int ena_parse_devargs(struct ena_adapter *adapter, 250 struct rte_devargs *devargs); 251 static int ena_copy_eni_stats(struct ena_adapter *adapter); 252 253 static const struct eth_dev_ops ena_dev_ops = { 254 .dev_configure = ena_dev_configure, 255 .dev_infos_get = ena_infos_get, 256 .rx_queue_setup = ena_rx_queue_setup, 257 .tx_queue_setup = ena_tx_queue_setup, 258 .dev_start = ena_start, 259 .dev_stop = ena_stop, 260 .link_update = ena_link_update, 261 .stats_get = ena_stats_get, 262 .xstats_get_names = ena_xstats_get_names, 263 .xstats_get = ena_xstats_get, 264 .xstats_get_by_id = ena_xstats_get_by_id, 265 .mtu_set = ena_mtu_set, 266 .rx_queue_release = ena_rx_queue_release, 267 .tx_queue_release = ena_tx_queue_release, 268 .dev_close = ena_close, 269 .dev_reset = ena_dev_reset, 270 .reta_update = ena_rss_reta_update, 271 .reta_query = ena_rss_reta_query, 272 }; 273 274 void ena_rss_key_fill(void *key, size_t size) 275 { 276 static bool key_generated; 277 static uint8_t default_key[ENA_HASH_KEY_SIZE]; 278 size_t i; 279 280 RTE_ASSERT(size <= ENA_HASH_KEY_SIZE); 281 282 if (!key_generated) { 283 for (i = 0; i < ENA_HASH_KEY_SIZE; ++i) 284 default_key[i] = rte_rand() & 0xff; 285 key_generated = true; 286 } 287 288 rte_memcpy(key, default_key, size); 289 } 290 291 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 292 struct ena_com_rx_ctx *ena_rx_ctx) 293 { 294 uint64_t ol_flags = 0; 295 uint32_t packet_type = 0; 296 297 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 298 packet_type |= RTE_PTYPE_L4_TCP; 299 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 300 packet_type |= RTE_PTYPE_L4_UDP; 301 302 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) { 303 packet_type |= RTE_PTYPE_L3_IPV4; 304 if (unlikely(ena_rx_ctx->l3_csum_err)) 305 ol_flags |= PKT_RX_IP_CKSUM_BAD; 306 else 307 ol_flags |= PKT_RX_IP_CKSUM_GOOD; 308 } else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) { 309 packet_type |= RTE_PTYPE_L3_IPV6; 310 } 311 312 if (!ena_rx_ctx->l4_csum_checked || ena_rx_ctx->frag) 313 ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 314 else 315 if (unlikely(ena_rx_ctx->l4_csum_err)) 316 ol_flags |= PKT_RX_L4_CKSUM_BAD; 317 else 318 ol_flags |= PKT_RX_L4_CKSUM_GOOD; 319 320 if (likely((packet_type & ENA_PTYPE_HAS_HASH) && !ena_rx_ctx->frag)) { 321 ol_flags |= PKT_RX_RSS_HASH; 322 mbuf->hash.rss = ena_rx_ctx->hash; 323 } 324 325 mbuf->ol_flags = ol_flags; 326 mbuf->packet_type = packet_type; 327 } 328 329 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 330 struct ena_com_tx_ctx *ena_tx_ctx, 331 uint64_t queue_offloads, 332 bool disable_meta_caching) 333 { 334 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 335 336 if ((mbuf->ol_flags & MBUF_OFFLOADS) && 337 (queue_offloads & QUEUE_OFFLOADS)) { 338 /* check if TSO is required */ 339 if ((mbuf->ol_flags & PKT_TX_TCP_SEG) && 340 (queue_offloads & DEV_TX_OFFLOAD_TCP_TSO)) { 341 ena_tx_ctx->tso_enable = true; 342 343 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 344 } 345 346 /* check if L3 checksum is needed */ 347 if ((mbuf->ol_flags & PKT_TX_IP_CKSUM) && 348 (queue_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)) 349 ena_tx_ctx->l3_csum_enable = true; 350 351 if (mbuf->ol_flags & PKT_TX_IPV6) { 352 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 353 } else { 354 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 355 356 /* set don't fragment (DF) flag */ 357 if (mbuf->packet_type & 358 (RTE_PTYPE_L4_NONFRAG 359 | RTE_PTYPE_INNER_L4_NONFRAG)) 360 ena_tx_ctx->df = true; 361 } 362 363 /* check if L4 checksum is needed */ 364 if (((mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) && 365 (queue_offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) { 366 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 367 ena_tx_ctx->l4_csum_enable = true; 368 } else if (((mbuf->ol_flags & PKT_TX_L4_MASK) == 369 PKT_TX_UDP_CKSUM) && 370 (queue_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) { 371 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 372 ena_tx_ctx->l4_csum_enable = true; 373 } else { 374 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 375 ena_tx_ctx->l4_csum_enable = false; 376 } 377 378 ena_meta->mss = mbuf->tso_segsz; 379 ena_meta->l3_hdr_len = mbuf->l3_len; 380 ena_meta->l3_hdr_offset = mbuf->l2_len; 381 382 ena_tx_ctx->meta_valid = true; 383 } else if (disable_meta_caching) { 384 memset(ena_meta, 0, sizeof(*ena_meta)); 385 ena_tx_ctx->meta_valid = true; 386 } else { 387 ena_tx_ctx->meta_valid = false; 388 } 389 } 390 391 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) 392 { 393 struct ena_tx_buffer *tx_info = NULL; 394 395 if (likely(req_id < tx_ring->ring_size)) { 396 tx_info = &tx_ring->tx_buffer_info[req_id]; 397 if (likely(tx_info->mbuf)) 398 return 0; 399 } 400 401 if (tx_info) 402 PMD_TX_LOG(ERR, "tx_info doesn't have valid mbuf\n"); 403 else 404 PMD_TX_LOG(ERR, "Invalid req_id: %hu\n", req_id); 405 406 /* Trigger device reset */ 407 ++tx_ring->tx_stats.bad_req_id; 408 tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; 409 tx_ring->adapter->trigger_reset = true; 410 return -EFAULT; 411 } 412 413 static void ena_config_host_info(struct ena_com_dev *ena_dev) 414 { 415 struct ena_admin_host_info *host_info; 416 int rc; 417 418 /* Allocate only the host info */ 419 rc = ena_com_allocate_host_info(ena_dev); 420 if (rc) { 421 PMD_DRV_LOG(ERR, "Cannot allocate host info\n"); 422 return; 423 } 424 425 host_info = ena_dev->host_attr.host_info; 426 427 host_info->os_type = ENA_ADMIN_OS_DPDK; 428 host_info->kernel_ver = RTE_VERSION; 429 strlcpy((char *)host_info->kernel_ver_str, rte_version(), 430 sizeof(host_info->kernel_ver_str)); 431 host_info->os_dist = RTE_VERSION; 432 strlcpy((char *)host_info->os_dist_str, rte_version(), 433 sizeof(host_info->os_dist_str)); 434 host_info->driver_version = 435 (DRV_MODULE_VER_MAJOR) | 436 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 437 (DRV_MODULE_VER_SUBMINOR << 438 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 439 host_info->num_cpus = rte_lcore_count(); 440 441 host_info->driver_supported_features = 442 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK; 443 444 rc = ena_com_set_host_attributes(ena_dev); 445 if (rc) { 446 if (rc == -ENA_COM_UNSUPPORTED) 447 PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); 448 else 449 PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); 450 451 goto err; 452 } 453 454 return; 455 456 err: 457 ena_com_delete_host_info(ena_dev); 458 } 459 460 /* This function calculates the number of xstats based on the current config */ 461 static unsigned int ena_xstats_calc_num(struct rte_eth_dev_data *data) 462 { 463 return ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENI + 464 (data->nb_tx_queues * ENA_STATS_ARRAY_TX) + 465 (data->nb_rx_queues * ENA_STATS_ARRAY_RX); 466 } 467 468 static void ena_config_debug_area(struct ena_adapter *adapter) 469 { 470 u32 debug_area_size; 471 int rc, ss_count; 472 473 ss_count = ena_xstats_calc_num(adapter->edev_data); 474 475 /* allocate 32 bytes for each string and 64bit for the value */ 476 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 477 478 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 479 if (rc) { 480 PMD_DRV_LOG(ERR, "Cannot allocate debug area\n"); 481 return; 482 } 483 484 rc = ena_com_set_host_attributes(&adapter->ena_dev); 485 if (rc) { 486 if (rc == -ENA_COM_UNSUPPORTED) 487 PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); 488 else 489 PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); 490 491 goto err; 492 } 493 494 return; 495 err: 496 ena_com_delete_debug_area(&adapter->ena_dev); 497 } 498 499 static int ena_close(struct rte_eth_dev *dev) 500 { 501 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 502 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 503 struct ena_adapter *adapter = dev->data->dev_private; 504 int ret = 0; 505 506 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 507 return 0; 508 509 if (adapter->state == ENA_ADAPTER_STATE_RUNNING) 510 ret = ena_stop(dev); 511 adapter->state = ENA_ADAPTER_STATE_CLOSED; 512 513 ena_rx_queue_release_all(dev); 514 ena_tx_queue_release_all(dev); 515 516 rte_free(adapter->drv_stats); 517 adapter->drv_stats = NULL; 518 519 rte_intr_disable(intr_handle); 520 rte_intr_callback_unregister(intr_handle, 521 ena_interrupt_handler_rte, 522 dev); 523 524 /* 525 * MAC is not allocated dynamically. Setting NULL should prevent from 526 * release of the resource in the rte_eth_dev_release_port(). 527 */ 528 dev->data->mac_addrs = NULL; 529 530 return ret; 531 } 532 533 static int 534 ena_dev_reset(struct rte_eth_dev *dev) 535 { 536 int rc = 0; 537 538 /* Cannot release memory in secondary process */ 539 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 540 PMD_DRV_LOG(WARNING, "dev_reset not supported in secondary.\n"); 541 return -EPERM; 542 } 543 544 ena_destroy_device(dev); 545 rc = eth_ena_dev_init(dev); 546 if (rc) 547 PMD_INIT_LOG(CRIT, "Cannot initialize device\n"); 548 549 return rc; 550 } 551 552 static int ena_rss_reta_update(struct rte_eth_dev *dev, 553 struct rte_eth_rss_reta_entry64 *reta_conf, 554 uint16_t reta_size) 555 { 556 struct ena_adapter *adapter = dev->data->dev_private; 557 struct ena_com_dev *ena_dev = &adapter->ena_dev; 558 int rc, i; 559 u16 entry_value; 560 int conf_idx; 561 int idx; 562 563 if ((reta_size == 0) || (reta_conf == NULL)) 564 return -EINVAL; 565 566 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 567 PMD_DRV_LOG(WARNING, 568 "Requested indirection table size (%d) is bigger than supported: %d\n", 569 reta_size, ENA_RX_RSS_TABLE_SIZE); 570 return -EINVAL; 571 } 572 573 for (i = 0 ; i < reta_size ; i++) { 574 /* each reta_conf is for 64 entries. 575 * to support 128 we use 2 conf of 64 576 */ 577 conf_idx = i / RTE_RETA_GROUP_SIZE; 578 idx = i % RTE_RETA_GROUP_SIZE; 579 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 580 entry_value = 581 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 582 583 rc = ena_com_indirect_table_fill_entry(ena_dev, 584 i, 585 entry_value); 586 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 587 PMD_DRV_LOG(ERR, 588 "Cannot fill indirect table\n"); 589 return rc; 590 } 591 } 592 } 593 594 rte_spinlock_lock(&adapter->admin_lock); 595 rc = ena_com_indirect_table_set(ena_dev); 596 rte_spinlock_unlock(&adapter->admin_lock); 597 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 598 PMD_DRV_LOG(ERR, "Cannot flush the indirect table\n"); 599 return rc; 600 } 601 602 PMD_DRV_LOG(DEBUG, "RSS configured %d entries for port %d\n", 603 reta_size, dev->data->port_id); 604 605 return 0; 606 } 607 608 /* Query redirection table. */ 609 static int ena_rss_reta_query(struct rte_eth_dev *dev, 610 struct rte_eth_rss_reta_entry64 *reta_conf, 611 uint16_t reta_size) 612 { 613 struct ena_adapter *adapter = dev->data->dev_private; 614 struct ena_com_dev *ena_dev = &adapter->ena_dev; 615 int rc; 616 int i; 617 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 618 int reta_conf_idx; 619 int reta_idx; 620 621 if (reta_size == 0 || reta_conf == NULL || 622 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 623 return -EINVAL; 624 625 rte_spinlock_lock(&adapter->admin_lock); 626 rc = ena_com_indirect_table_get(ena_dev, indirect_table); 627 rte_spinlock_unlock(&adapter->admin_lock); 628 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 629 PMD_DRV_LOG(ERR, "Cannot get indirection table\n"); 630 return -ENOTSUP; 631 } 632 633 for (i = 0 ; i < reta_size ; i++) { 634 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 635 reta_idx = i % RTE_RETA_GROUP_SIZE; 636 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 637 reta_conf[reta_conf_idx].reta[reta_idx] = 638 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 639 } 640 641 return 0; 642 } 643 644 static int ena_rss_init_default(struct ena_adapter *adapter) 645 { 646 struct ena_com_dev *ena_dev = &adapter->ena_dev; 647 uint16_t nb_rx_queues = adapter->edev_data->nb_rx_queues; 648 int rc, i; 649 u32 val; 650 651 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 652 if (unlikely(rc)) { 653 PMD_DRV_LOG(ERR, "Cannot init indirection table\n"); 654 goto err_rss_init; 655 } 656 657 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 658 val = i % nb_rx_queues; 659 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 660 ENA_IO_RXQ_IDX(val)); 661 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 662 PMD_DRV_LOG(ERR, "Cannot fill indirection table\n"); 663 goto err_fill_indir; 664 } 665 } 666 667 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 668 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 669 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 670 PMD_DRV_LOG(INFO, "Cannot fill hash function\n"); 671 goto err_fill_indir; 672 } 673 674 rc = ena_com_set_default_hash_ctrl(ena_dev); 675 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 676 PMD_DRV_LOG(INFO, "Cannot fill hash control\n"); 677 goto err_fill_indir; 678 } 679 680 rc = ena_com_indirect_table_set(ena_dev); 681 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 682 PMD_DRV_LOG(ERR, "Cannot flush indirection table\n"); 683 goto err_fill_indir; 684 } 685 PMD_DRV_LOG(DEBUG, "RSS configured for port %d\n", 686 adapter->edev_data->port_id); 687 688 return 0; 689 690 err_fill_indir: 691 ena_com_rss_destroy(ena_dev); 692 err_rss_init: 693 694 return rc; 695 } 696 697 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 698 { 699 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 700 int nb_queues = dev->data->nb_rx_queues; 701 int i; 702 703 for (i = 0; i < nb_queues; i++) 704 ena_rx_queue_release(queues[i]); 705 } 706 707 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 708 { 709 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 710 int nb_queues = dev->data->nb_tx_queues; 711 int i; 712 713 for (i = 0; i < nb_queues; i++) 714 ena_tx_queue_release(queues[i]); 715 } 716 717 static void ena_rx_queue_release(void *queue) 718 { 719 struct ena_ring *ring = (struct ena_ring *)queue; 720 721 /* Free ring resources */ 722 if (ring->rx_buffer_info) 723 rte_free(ring->rx_buffer_info); 724 ring->rx_buffer_info = NULL; 725 726 if (ring->rx_refill_buffer) 727 rte_free(ring->rx_refill_buffer); 728 ring->rx_refill_buffer = NULL; 729 730 if (ring->empty_rx_reqs) 731 rte_free(ring->empty_rx_reqs); 732 ring->empty_rx_reqs = NULL; 733 734 ring->configured = 0; 735 736 PMD_DRV_LOG(NOTICE, "Rx queue %d:%d released\n", 737 ring->port_id, ring->id); 738 } 739 740 static void ena_tx_queue_release(void *queue) 741 { 742 struct ena_ring *ring = (struct ena_ring *)queue; 743 744 /* Free ring resources */ 745 if (ring->push_buf_intermediate_buf) 746 rte_free(ring->push_buf_intermediate_buf); 747 748 if (ring->tx_buffer_info) 749 rte_free(ring->tx_buffer_info); 750 751 if (ring->empty_tx_reqs) 752 rte_free(ring->empty_tx_reqs); 753 754 ring->empty_tx_reqs = NULL; 755 ring->tx_buffer_info = NULL; 756 ring->push_buf_intermediate_buf = NULL; 757 758 ring->configured = 0; 759 760 PMD_DRV_LOG(NOTICE, "Tx queue %d:%d released\n", 761 ring->port_id, ring->id); 762 } 763 764 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 765 { 766 unsigned int i; 767 768 for (i = 0; i < ring->ring_size; ++i) { 769 struct ena_rx_buffer *rx_info = &ring->rx_buffer_info[i]; 770 if (rx_info->mbuf) { 771 rte_mbuf_raw_free(rx_info->mbuf); 772 rx_info->mbuf = NULL; 773 } 774 } 775 } 776 777 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 778 { 779 unsigned int i; 780 781 for (i = 0; i < ring->ring_size; ++i) { 782 struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; 783 784 if (tx_buf->mbuf) { 785 rte_pktmbuf_free(tx_buf->mbuf); 786 tx_buf->mbuf = NULL; 787 } 788 } 789 } 790 791 static int ena_link_update(struct rte_eth_dev *dev, 792 __rte_unused int wait_to_complete) 793 { 794 struct rte_eth_link *link = &dev->data->dev_link; 795 struct ena_adapter *adapter = dev->data->dev_private; 796 797 link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN; 798 link->link_speed = ETH_SPEED_NUM_NONE; 799 link->link_duplex = ETH_LINK_FULL_DUPLEX; 800 801 return 0; 802 } 803 804 static int ena_queue_start_all(struct rte_eth_dev *dev, 805 enum ena_ring_type ring_type) 806 { 807 struct ena_adapter *adapter = dev->data->dev_private; 808 struct ena_ring *queues = NULL; 809 int nb_queues; 810 int i = 0; 811 int rc = 0; 812 813 if (ring_type == ENA_RING_TYPE_RX) { 814 queues = adapter->rx_ring; 815 nb_queues = dev->data->nb_rx_queues; 816 } else { 817 queues = adapter->tx_ring; 818 nb_queues = dev->data->nb_tx_queues; 819 } 820 for (i = 0; i < nb_queues; i++) { 821 if (queues[i].configured) { 822 if (ring_type == ENA_RING_TYPE_RX) { 823 ena_assert_msg( 824 dev->data->rx_queues[i] == &queues[i], 825 "Inconsistent state of Rx queues\n"); 826 } else { 827 ena_assert_msg( 828 dev->data->tx_queues[i] == &queues[i], 829 "Inconsistent state of Tx queues\n"); 830 } 831 832 rc = ena_queue_start(&queues[i]); 833 834 if (rc) { 835 PMD_INIT_LOG(ERR, 836 "Failed to start queue[%d] of type(%d)\n", 837 i, ring_type); 838 goto err; 839 } 840 } 841 } 842 843 return 0; 844 845 err: 846 while (i--) 847 if (queues[i].configured) 848 ena_queue_stop(&queues[i]); 849 850 return rc; 851 } 852 853 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 854 { 855 uint32_t max_frame_len = adapter->max_mtu; 856 857 if (adapter->edev_data->dev_conf.rxmode.offloads & 858 DEV_RX_OFFLOAD_JUMBO_FRAME) 859 max_frame_len = 860 adapter->edev_data->dev_conf.rxmode.max_rx_pkt_len; 861 862 return max_frame_len; 863 } 864 865 static int ena_check_valid_conf(struct ena_adapter *adapter) 866 { 867 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 868 869 if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) { 870 PMD_INIT_LOG(ERR, 871 "Unsupported MTU of %d. Max MTU: %d, min MTU: %d\n", 872 max_frame_len, adapter->max_mtu, ENA_MIN_MTU); 873 return ENA_COM_UNSUPPORTED; 874 } 875 876 return 0; 877 } 878 879 static int 880 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx, 881 bool use_large_llq_hdr) 882 { 883 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; 884 struct ena_com_dev *ena_dev = ctx->ena_dev; 885 uint32_t max_tx_queue_size; 886 uint32_t max_rx_queue_size; 887 888 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 889 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 890 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; 891 max_rx_queue_size = RTE_MIN(max_queue_ext->max_rx_cq_depth, 892 max_queue_ext->max_rx_sq_depth); 893 max_tx_queue_size = max_queue_ext->max_tx_cq_depth; 894 895 if (ena_dev->tx_mem_queue_type == 896 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 897 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 898 llq->max_llq_depth); 899 } else { 900 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 901 max_queue_ext->max_tx_sq_depth); 902 } 903 904 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 905 max_queue_ext->max_per_packet_rx_descs); 906 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 907 max_queue_ext->max_per_packet_tx_descs); 908 } else { 909 struct ena_admin_queue_feature_desc *max_queues = 910 &ctx->get_feat_ctx->max_queues; 911 max_rx_queue_size = RTE_MIN(max_queues->max_cq_depth, 912 max_queues->max_sq_depth); 913 max_tx_queue_size = max_queues->max_cq_depth; 914 915 if (ena_dev->tx_mem_queue_type == 916 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 917 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 918 llq->max_llq_depth); 919 } else { 920 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 921 max_queues->max_sq_depth); 922 } 923 924 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 925 max_queues->max_packet_rx_descs); 926 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 927 max_queues->max_packet_tx_descs); 928 } 929 930 /* Round down to the nearest power of 2 */ 931 max_rx_queue_size = rte_align32prevpow2(max_rx_queue_size); 932 max_tx_queue_size = rte_align32prevpow2(max_tx_queue_size); 933 934 if (use_large_llq_hdr) { 935 if ((llq->entry_size_ctrl_supported & 936 ENA_ADMIN_LIST_ENTRY_SIZE_256B) && 937 (ena_dev->tx_mem_queue_type == 938 ENA_ADMIN_PLACEMENT_POLICY_DEV)) { 939 max_tx_queue_size /= 2; 940 PMD_INIT_LOG(INFO, 941 "Forcing large headers and decreasing maximum Tx queue size to %d\n", 942 max_tx_queue_size); 943 } else { 944 PMD_INIT_LOG(ERR, 945 "Forcing large headers failed: LLQ is disabled or device does not support large headers\n"); 946 } 947 } 948 949 if (unlikely(max_rx_queue_size == 0 || max_tx_queue_size == 0)) { 950 PMD_INIT_LOG(ERR, "Invalid queue size\n"); 951 return -EFAULT; 952 } 953 954 ctx->max_tx_queue_size = max_tx_queue_size; 955 ctx->max_rx_queue_size = max_rx_queue_size; 956 957 return 0; 958 } 959 960 static void ena_stats_restart(struct rte_eth_dev *dev) 961 { 962 struct ena_adapter *adapter = dev->data->dev_private; 963 964 rte_atomic64_init(&adapter->drv_stats->ierrors); 965 rte_atomic64_init(&adapter->drv_stats->oerrors); 966 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 967 adapter->drv_stats->rx_drops = 0; 968 } 969 970 static int ena_stats_get(struct rte_eth_dev *dev, 971 struct rte_eth_stats *stats) 972 { 973 struct ena_admin_basic_stats ena_stats; 974 struct ena_adapter *adapter = dev->data->dev_private; 975 struct ena_com_dev *ena_dev = &adapter->ena_dev; 976 int rc; 977 int i; 978 int max_rings_stats; 979 980 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 981 return -ENOTSUP; 982 983 memset(&ena_stats, 0, sizeof(ena_stats)); 984 985 rte_spinlock_lock(&adapter->admin_lock); 986 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 987 rte_spinlock_unlock(&adapter->admin_lock); 988 if (unlikely(rc)) { 989 PMD_DRV_LOG(ERR, "Could not retrieve statistics from ENA\n"); 990 return rc; 991 } 992 993 /* Set of basic statistics from ENA */ 994 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 995 ena_stats.rx_pkts_low); 996 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 997 ena_stats.tx_pkts_low); 998 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 999 ena_stats.rx_bytes_low); 1000 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 1001 ena_stats.tx_bytes_low); 1002 1003 /* Driver related stats */ 1004 stats->imissed = adapter->drv_stats->rx_drops; 1005 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 1006 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 1007 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 1008 1009 max_rings_stats = RTE_MIN(dev->data->nb_rx_queues, 1010 RTE_ETHDEV_QUEUE_STAT_CNTRS); 1011 for (i = 0; i < max_rings_stats; ++i) { 1012 struct ena_stats_rx *rx_stats = &adapter->rx_ring[i].rx_stats; 1013 1014 stats->q_ibytes[i] = rx_stats->bytes; 1015 stats->q_ipackets[i] = rx_stats->cnt; 1016 stats->q_errors[i] = rx_stats->bad_desc_num + 1017 rx_stats->bad_req_id; 1018 } 1019 1020 max_rings_stats = RTE_MIN(dev->data->nb_tx_queues, 1021 RTE_ETHDEV_QUEUE_STAT_CNTRS); 1022 for (i = 0; i < max_rings_stats; ++i) { 1023 struct ena_stats_tx *tx_stats = &adapter->tx_ring[i].tx_stats; 1024 1025 stats->q_obytes[i] = tx_stats->bytes; 1026 stats->q_opackets[i] = tx_stats->cnt; 1027 } 1028 1029 return 0; 1030 } 1031 1032 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1033 { 1034 struct ena_adapter *adapter; 1035 struct ena_com_dev *ena_dev; 1036 int rc = 0; 1037 1038 ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); 1039 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); 1040 adapter = dev->data->dev_private; 1041 1042 ena_dev = &adapter->ena_dev; 1043 ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); 1044 1045 if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) { 1046 PMD_DRV_LOG(ERR, 1047 "Invalid MTU setting. New MTU: %d, max MTU: %d, min MTU: %d\n", 1048 mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU); 1049 return -EINVAL; 1050 } 1051 1052 rc = ena_com_set_dev_mtu(ena_dev, mtu); 1053 if (rc) 1054 PMD_DRV_LOG(ERR, "Could not set MTU: %d\n", mtu); 1055 else 1056 PMD_DRV_LOG(NOTICE, "MTU set to: %d\n", mtu); 1057 1058 return rc; 1059 } 1060 1061 static int ena_start(struct rte_eth_dev *dev) 1062 { 1063 struct ena_adapter *adapter = dev->data->dev_private; 1064 uint64_t ticks; 1065 int rc = 0; 1066 1067 /* Cannot allocate memory in secondary process */ 1068 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1069 PMD_DRV_LOG(WARNING, "dev_start not supported in secondary.\n"); 1070 return -EPERM; 1071 } 1072 1073 rc = ena_check_valid_conf(adapter); 1074 if (rc) 1075 return rc; 1076 1077 rc = ena_queue_start_all(dev, ENA_RING_TYPE_RX); 1078 if (rc) 1079 return rc; 1080 1081 rc = ena_queue_start_all(dev, ENA_RING_TYPE_TX); 1082 if (rc) 1083 goto err_start_tx; 1084 1085 if (adapter->edev_data->dev_conf.rxmode.mq_mode & 1086 ETH_MQ_RX_RSS_FLAG && adapter->edev_data->nb_rx_queues > 0) { 1087 rc = ena_rss_init_default(adapter); 1088 if (rc) 1089 goto err_rss_init; 1090 } 1091 1092 ena_stats_restart(dev); 1093 1094 adapter->timestamp_wd = rte_get_timer_cycles(); 1095 adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; 1096 1097 ticks = rte_get_timer_hz(); 1098 rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(), 1099 ena_timer_wd_callback, dev); 1100 1101 ++adapter->dev_stats.dev_start; 1102 adapter->state = ENA_ADAPTER_STATE_RUNNING; 1103 1104 return 0; 1105 1106 err_rss_init: 1107 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1108 err_start_tx: 1109 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1110 return rc; 1111 } 1112 1113 static int ena_stop(struct rte_eth_dev *dev) 1114 { 1115 struct ena_adapter *adapter = dev->data->dev_private; 1116 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1117 int rc; 1118 1119 /* Cannot free memory in secondary process */ 1120 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1121 PMD_DRV_LOG(WARNING, "dev_stop not supported in secondary.\n"); 1122 return -EPERM; 1123 } 1124 1125 rte_timer_stop_sync(&adapter->timer_wd); 1126 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1127 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1128 1129 if (adapter->trigger_reset) { 1130 rc = ena_com_dev_reset(ena_dev, adapter->reset_reason); 1131 if (rc) 1132 PMD_DRV_LOG(ERR, "Device reset failed, rc: %d\n", rc); 1133 } 1134 1135 ++adapter->dev_stats.dev_stop; 1136 adapter->state = ENA_ADAPTER_STATE_STOPPED; 1137 dev->data->dev_started = 0; 1138 1139 return 0; 1140 } 1141 1142 static int ena_create_io_queue(struct ena_ring *ring) 1143 { 1144 struct ena_adapter *adapter; 1145 struct ena_com_dev *ena_dev; 1146 struct ena_com_create_io_ctx ctx = 1147 /* policy set to _HOST just to satisfy icc compiler */ 1148 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1149 0, 0, 0, 0, 0 }; 1150 uint16_t ena_qid; 1151 unsigned int i; 1152 int rc; 1153 1154 adapter = ring->adapter; 1155 ena_dev = &adapter->ena_dev; 1156 1157 if (ring->type == ENA_RING_TYPE_TX) { 1158 ena_qid = ENA_IO_TXQ_IDX(ring->id); 1159 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1160 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1161 for (i = 0; i < ring->ring_size; i++) 1162 ring->empty_tx_reqs[i] = i; 1163 } else { 1164 ena_qid = ENA_IO_RXQ_IDX(ring->id); 1165 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1166 for (i = 0; i < ring->ring_size; i++) 1167 ring->empty_rx_reqs[i] = i; 1168 } 1169 ctx.queue_size = ring->ring_size; 1170 ctx.qid = ena_qid; 1171 ctx.msix_vector = -1; /* interrupts not used */ 1172 ctx.numa_node = ring->numa_socket_id; 1173 1174 rc = ena_com_create_io_queue(ena_dev, &ctx); 1175 if (rc) { 1176 PMD_DRV_LOG(ERR, 1177 "Failed to create IO queue[%d] (qid:%d), rc: %d\n", 1178 ring->id, ena_qid, rc); 1179 return rc; 1180 } 1181 1182 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1183 &ring->ena_com_io_sq, 1184 &ring->ena_com_io_cq); 1185 if (rc) { 1186 PMD_DRV_LOG(ERR, 1187 "Failed to get IO queue[%d] handlers, rc: %d\n", 1188 ring->id, rc); 1189 ena_com_destroy_io_queue(ena_dev, ena_qid); 1190 return rc; 1191 } 1192 1193 if (ring->type == ENA_RING_TYPE_TX) 1194 ena_com_update_numa_node(ring->ena_com_io_cq, ctx.numa_node); 1195 1196 return 0; 1197 } 1198 1199 static void ena_queue_stop(struct ena_ring *ring) 1200 { 1201 struct ena_com_dev *ena_dev = &ring->adapter->ena_dev; 1202 1203 if (ring->type == ENA_RING_TYPE_RX) { 1204 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(ring->id)); 1205 ena_rx_queue_release_bufs(ring); 1206 } else { 1207 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(ring->id)); 1208 ena_tx_queue_release_bufs(ring); 1209 } 1210 } 1211 1212 static void ena_queue_stop_all(struct rte_eth_dev *dev, 1213 enum ena_ring_type ring_type) 1214 { 1215 struct ena_adapter *adapter = dev->data->dev_private; 1216 struct ena_ring *queues = NULL; 1217 uint16_t nb_queues, i; 1218 1219 if (ring_type == ENA_RING_TYPE_RX) { 1220 queues = adapter->rx_ring; 1221 nb_queues = dev->data->nb_rx_queues; 1222 } else { 1223 queues = adapter->tx_ring; 1224 nb_queues = dev->data->nb_tx_queues; 1225 } 1226 1227 for (i = 0; i < nb_queues; ++i) 1228 if (queues[i].configured) 1229 ena_queue_stop(&queues[i]); 1230 } 1231 1232 static int ena_queue_start(struct ena_ring *ring) 1233 { 1234 int rc, bufs_num; 1235 1236 ena_assert_msg(ring->configured == 1, 1237 "Trying to start unconfigured queue\n"); 1238 1239 rc = ena_create_io_queue(ring); 1240 if (rc) { 1241 PMD_INIT_LOG(ERR, "Failed to create IO queue\n"); 1242 return rc; 1243 } 1244 1245 ring->next_to_clean = 0; 1246 ring->next_to_use = 0; 1247 1248 if (ring->type == ENA_RING_TYPE_TX) { 1249 ring->tx_stats.available_desc = 1250 ena_com_free_q_entries(ring->ena_com_io_sq); 1251 return 0; 1252 } 1253 1254 bufs_num = ring->ring_size - 1; 1255 rc = ena_populate_rx_queue(ring, bufs_num); 1256 if (rc != bufs_num) { 1257 ena_com_destroy_io_queue(&ring->adapter->ena_dev, 1258 ENA_IO_RXQ_IDX(ring->id)); 1259 PMD_INIT_LOG(ERR, "Failed to populate Rx ring\n"); 1260 return ENA_COM_FAULT; 1261 } 1262 /* Flush per-core RX buffers pools cache as they can be used on other 1263 * cores as well. 1264 */ 1265 rte_mempool_cache_flush(NULL, ring->mb_pool); 1266 1267 return 0; 1268 } 1269 1270 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 1271 uint16_t queue_idx, 1272 uint16_t nb_desc, 1273 unsigned int socket_id, 1274 const struct rte_eth_txconf *tx_conf) 1275 { 1276 struct ena_ring *txq = NULL; 1277 struct ena_adapter *adapter = dev->data->dev_private; 1278 unsigned int i; 1279 1280 txq = &adapter->tx_ring[queue_idx]; 1281 1282 if (txq->configured) { 1283 PMD_DRV_LOG(CRIT, 1284 "API violation. Queue[%d] is already configured\n", 1285 queue_idx); 1286 return ENA_COM_FAULT; 1287 } 1288 1289 if (!rte_is_power_of_2(nb_desc)) { 1290 PMD_DRV_LOG(ERR, 1291 "Unsupported size of Tx queue: %d is not a power of 2.\n", 1292 nb_desc); 1293 return -EINVAL; 1294 } 1295 1296 if (nb_desc > adapter->max_tx_ring_size) { 1297 PMD_DRV_LOG(ERR, 1298 "Unsupported size of Tx queue (max size: %d)\n", 1299 adapter->max_tx_ring_size); 1300 return -EINVAL; 1301 } 1302 1303 txq->port_id = dev->data->port_id; 1304 txq->next_to_clean = 0; 1305 txq->next_to_use = 0; 1306 txq->ring_size = nb_desc; 1307 txq->size_mask = nb_desc - 1; 1308 txq->numa_socket_id = socket_id; 1309 txq->pkts_without_db = false; 1310 1311 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1312 sizeof(struct ena_tx_buffer) * 1313 txq->ring_size, 1314 RTE_CACHE_LINE_SIZE); 1315 if (!txq->tx_buffer_info) { 1316 PMD_DRV_LOG(ERR, 1317 "Failed to allocate memory for Tx buffer info\n"); 1318 return -ENOMEM; 1319 } 1320 1321 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1322 sizeof(u16) * txq->ring_size, 1323 RTE_CACHE_LINE_SIZE); 1324 if (!txq->empty_tx_reqs) { 1325 PMD_DRV_LOG(ERR, 1326 "Failed to allocate memory for empty Tx requests\n"); 1327 rte_free(txq->tx_buffer_info); 1328 return -ENOMEM; 1329 } 1330 1331 txq->push_buf_intermediate_buf = 1332 rte_zmalloc("txq->push_buf_intermediate_buf", 1333 txq->tx_max_header_size, 1334 RTE_CACHE_LINE_SIZE); 1335 if (!txq->push_buf_intermediate_buf) { 1336 PMD_DRV_LOG(ERR, "Failed to alloc push buffer for LLQ\n"); 1337 rte_free(txq->tx_buffer_info); 1338 rte_free(txq->empty_tx_reqs); 1339 return -ENOMEM; 1340 } 1341 1342 for (i = 0; i < txq->ring_size; i++) 1343 txq->empty_tx_reqs[i] = i; 1344 1345 if (tx_conf != NULL) { 1346 txq->offloads = 1347 tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1348 } 1349 /* Store pointer to this queue in upper layer */ 1350 txq->configured = 1; 1351 dev->data->tx_queues[queue_idx] = txq; 1352 1353 return 0; 1354 } 1355 1356 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1357 uint16_t queue_idx, 1358 uint16_t nb_desc, 1359 unsigned int socket_id, 1360 __rte_unused const struct rte_eth_rxconf *rx_conf, 1361 struct rte_mempool *mp) 1362 { 1363 struct ena_adapter *adapter = dev->data->dev_private; 1364 struct ena_ring *rxq = NULL; 1365 size_t buffer_size; 1366 int i; 1367 1368 rxq = &adapter->rx_ring[queue_idx]; 1369 if (rxq->configured) { 1370 PMD_DRV_LOG(CRIT, 1371 "API violation. Queue[%d] is already configured\n", 1372 queue_idx); 1373 return ENA_COM_FAULT; 1374 } 1375 1376 if (!rte_is_power_of_2(nb_desc)) { 1377 PMD_DRV_LOG(ERR, 1378 "Unsupported size of Rx queue: %d is not a power of 2.\n", 1379 nb_desc); 1380 return -EINVAL; 1381 } 1382 1383 if (nb_desc > adapter->max_rx_ring_size) { 1384 PMD_DRV_LOG(ERR, 1385 "Unsupported size of Rx queue (max size: %d)\n", 1386 adapter->max_rx_ring_size); 1387 return -EINVAL; 1388 } 1389 1390 /* ENA isn't supporting buffers smaller than 1400 bytes */ 1391 buffer_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM; 1392 if (buffer_size < ENA_RX_BUF_MIN_SIZE) { 1393 PMD_DRV_LOG(ERR, 1394 "Unsupported size of Rx buffer: %zu (min size: %d)\n", 1395 buffer_size, ENA_RX_BUF_MIN_SIZE); 1396 return -EINVAL; 1397 } 1398 1399 rxq->port_id = dev->data->port_id; 1400 rxq->next_to_clean = 0; 1401 rxq->next_to_use = 0; 1402 rxq->ring_size = nb_desc; 1403 rxq->size_mask = nb_desc - 1; 1404 rxq->numa_socket_id = socket_id; 1405 rxq->mb_pool = mp; 1406 1407 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1408 sizeof(struct ena_rx_buffer) * nb_desc, 1409 RTE_CACHE_LINE_SIZE); 1410 if (!rxq->rx_buffer_info) { 1411 PMD_DRV_LOG(ERR, 1412 "Failed to allocate memory for Rx buffer info\n"); 1413 return -ENOMEM; 1414 } 1415 1416 rxq->rx_refill_buffer = rte_zmalloc("rxq->rx_refill_buffer", 1417 sizeof(struct rte_mbuf *) * nb_desc, 1418 RTE_CACHE_LINE_SIZE); 1419 1420 if (!rxq->rx_refill_buffer) { 1421 PMD_DRV_LOG(ERR, 1422 "Failed to allocate memory for Rx refill buffer\n"); 1423 rte_free(rxq->rx_buffer_info); 1424 rxq->rx_buffer_info = NULL; 1425 return -ENOMEM; 1426 } 1427 1428 rxq->empty_rx_reqs = rte_zmalloc("rxq->empty_rx_reqs", 1429 sizeof(uint16_t) * nb_desc, 1430 RTE_CACHE_LINE_SIZE); 1431 if (!rxq->empty_rx_reqs) { 1432 PMD_DRV_LOG(ERR, 1433 "Failed to allocate memory for empty Rx requests\n"); 1434 rte_free(rxq->rx_buffer_info); 1435 rxq->rx_buffer_info = NULL; 1436 rte_free(rxq->rx_refill_buffer); 1437 rxq->rx_refill_buffer = NULL; 1438 return -ENOMEM; 1439 } 1440 1441 for (i = 0; i < nb_desc; i++) 1442 rxq->empty_rx_reqs[i] = i; 1443 1444 /* Store pointer to this queue in upper layer */ 1445 rxq->configured = 1; 1446 dev->data->rx_queues[queue_idx] = rxq; 1447 1448 return 0; 1449 } 1450 1451 static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, 1452 struct rte_mbuf *mbuf, uint16_t id) 1453 { 1454 struct ena_com_buf ebuf; 1455 int rc; 1456 1457 /* prepare physical address for DMA transaction */ 1458 ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; 1459 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1460 1461 /* pass resource to device */ 1462 rc = ena_com_add_single_rx_desc(io_sq, &ebuf, id); 1463 if (unlikely(rc != 0)) 1464 PMD_RX_LOG(WARNING, "Failed adding Rx desc\n"); 1465 1466 return rc; 1467 } 1468 1469 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1470 { 1471 unsigned int i; 1472 int rc; 1473 uint16_t next_to_use = rxq->next_to_use; 1474 uint16_t req_id; 1475 #ifdef RTE_ETHDEV_DEBUG_RX 1476 uint16_t in_use; 1477 #endif 1478 struct rte_mbuf **mbufs = rxq->rx_refill_buffer; 1479 1480 if (unlikely(!count)) 1481 return 0; 1482 1483 #ifdef RTE_ETHDEV_DEBUG_RX 1484 in_use = rxq->ring_size - 1 - 1485 ena_com_free_q_entries(rxq->ena_com_io_sq); 1486 if (unlikely((in_use + count) >= rxq->ring_size)) 1487 PMD_RX_LOG(ERR, "Bad Rx ring state\n"); 1488 #endif 1489 1490 /* get resources for incoming packets */ 1491 rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, count); 1492 if (unlikely(rc < 0)) { 1493 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1494 ++rxq->rx_stats.mbuf_alloc_fail; 1495 PMD_RX_LOG(DEBUG, "There are not enough free buffers\n"); 1496 return 0; 1497 } 1498 1499 for (i = 0; i < count; i++) { 1500 struct rte_mbuf *mbuf = mbufs[i]; 1501 struct ena_rx_buffer *rx_info; 1502 1503 if (likely((i + 4) < count)) 1504 rte_prefetch0(mbufs[i + 4]); 1505 1506 req_id = rxq->empty_rx_reqs[next_to_use]; 1507 rx_info = &rxq->rx_buffer_info[req_id]; 1508 1509 rc = ena_add_single_rx_desc(rxq->ena_com_io_sq, mbuf, req_id); 1510 if (unlikely(rc != 0)) 1511 break; 1512 1513 rx_info->mbuf = mbuf; 1514 next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, rxq->size_mask); 1515 } 1516 1517 if (unlikely(i < count)) { 1518 PMD_RX_LOG(WARNING, 1519 "Refilled Rx queue[%d] with only %d/%d buffers\n", 1520 rxq->id, i, count); 1521 rte_pktmbuf_free_bulk(&mbufs[i], count - i); 1522 ++rxq->rx_stats.refill_partial; 1523 } 1524 1525 /* When we submitted free recources to device... */ 1526 if (likely(i > 0)) { 1527 /* ...let HW know that it can fill buffers with data. */ 1528 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1529 1530 rxq->next_to_use = next_to_use; 1531 } 1532 1533 return i; 1534 } 1535 1536 static int ena_device_init(struct ena_com_dev *ena_dev, 1537 struct rte_pci_device *pdev, 1538 struct ena_com_dev_get_features_ctx *get_feat_ctx, 1539 bool *wd_state) 1540 { 1541 uint32_t aenq_groups; 1542 int rc; 1543 bool readless_supported; 1544 1545 /* Initialize mmio registers */ 1546 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1547 if (rc) { 1548 PMD_DRV_LOG(ERR, "Failed to init MMIO read less\n"); 1549 return rc; 1550 } 1551 1552 /* The PCIe configuration space revision id indicate if mmio reg 1553 * read is disabled. 1554 */ 1555 readless_supported = !(pdev->id.class_id & ENA_MMIO_DISABLE_REG_READ); 1556 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1557 1558 /* reset device */ 1559 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 1560 if (rc) { 1561 PMD_DRV_LOG(ERR, "Cannot reset device\n"); 1562 goto err_mmio_read_less; 1563 } 1564 1565 /* check FW version */ 1566 rc = ena_com_validate_version(ena_dev); 1567 if (rc) { 1568 PMD_DRV_LOG(ERR, "Device version is too low\n"); 1569 goto err_mmio_read_less; 1570 } 1571 1572 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1573 1574 /* ENA device administration layer init */ 1575 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 1576 if (rc) { 1577 PMD_DRV_LOG(ERR, 1578 "Cannot initialize ENA admin queue\n"); 1579 goto err_mmio_read_less; 1580 } 1581 1582 /* To enable the msix interrupts the driver needs to know the number 1583 * of queues. So the driver uses polling mode to retrieve this 1584 * information. 1585 */ 1586 ena_com_set_admin_polling_mode(ena_dev, true); 1587 1588 ena_config_host_info(ena_dev); 1589 1590 /* Get Device Attributes and features */ 1591 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1592 if (rc) { 1593 PMD_DRV_LOG(ERR, 1594 "Cannot get attribute for ENA device, rc: %d\n", rc); 1595 goto err_admin_init; 1596 } 1597 1598 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 1599 BIT(ENA_ADMIN_NOTIFICATION) | 1600 BIT(ENA_ADMIN_KEEP_ALIVE) | 1601 BIT(ENA_ADMIN_FATAL_ERROR) | 1602 BIT(ENA_ADMIN_WARNING); 1603 1604 aenq_groups &= get_feat_ctx->aenq.supported_groups; 1605 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 1606 if (rc) { 1607 PMD_DRV_LOG(ERR, "Cannot configure AENQ groups, rc: %d\n", rc); 1608 goto err_admin_init; 1609 } 1610 1611 *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 1612 1613 return 0; 1614 1615 err_admin_init: 1616 ena_com_admin_destroy(ena_dev); 1617 1618 err_mmio_read_less: 1619 ena_com_mmio_reg_read_request_destroy(ena_dev); 1620 1621 return rc; 1622 } 1623 1624 static void ena_interrupt_handler_rte(void *cb_arg) 1625 { 1626 struct rte_eth_dev *dev = cb_arg; 1627 struct ena_adapter *adapter = dev->data->dev_private; 1628 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1629 1630 ena_com_admin_q_comp_intr_handler(ena_dev); 1631 if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED)) 1632 ena_com_aenq_intr_handler(ena_dev, dev); 1633 } 1634 1635 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 1636 { 1637 if (!adapter->wd_state) 1638 return; 1639 1640 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 1641 return; 1642 1643 if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >= 1644 adapter->keep_alive_timeout)) { 1645 PMD_DRV_LOG(ERR, "Keep alive timeout\n"); 1646 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; 1647 adapter->trigger_reset = true; 1648 ++adapter->dev_stats.wd_expired; 1649 } 1650 } 1651 1652 /* Check if admin queue is enabled */ 1653 static void check_for_admin_com_state(struct ena_adapter *adapter) 1654 { 1655 if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) { 1656 PMD_DRV_LOG(ERR, "ENA admin queue is not in running state\n"); 1657 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; 1658 adapter->trigger_reset = true; 1659 } 1660 } 1661 1662 static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer, 1663 void *arg) 1664 { 1665 struct rte_eth_dev *dev = arg; 1666 struct ena_adapter *adapter = dev->data->dev_private; 1667 1668 check_for_missing_keep_alive(adapter); 1669 check_for_admin_com_state(adapter); 1670 1671 if (unlikely(adapter->trigger_reset)) { 1672 PMD_DRV_LOG(ERR, "Trigger reset is on\n"); 1673 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 1674 NULL); 1675 } 1676 } 1677 1678 static inline void 1679 set_default_llq_configurations(struct ena_llq_configurations *llq_config, 1680 struct ena_admin_feature_llq_desc *llq, 1681 bool use_large_llq_hdr) 1682 { 1683 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 1684 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 1685 llq_config->llq_num_decs_before_header = 1686 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 1687 1688 if (use_large_llq_hdr && 1689 (llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B)) { 1690 llq_config->llq_ring_entry_size = 1691 ENA_ADMIN_LIST_ENTRY_SIZE_256B; 1692 llq_config->llq_ring_entry_size_value = 256; 1693 } else { 1694 llq_config->llq_ring_entry_size = 1695 ENA_ADMIN_LIST_ENTRY_SIZE_128B; 1696 llq_config->llq_ring_entry_size_value = 128; 1697 } 1698 } 1699 1700 static int 1701 ena_set_queues_placement_policy(struct ena_adapter *adapter, 1702 struct ena_com_dev *ena_dev, 1703 struct ena_admin_feature_llq_desc *llq, 1704 struct ena_llq_configurations *llq_default_configurations) 1705 { 1706 int rc; 1707 u32 llq_feature_mask; 1708 1709 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 1710 if (!(ena_dev->supported_features & llq_feature_mask)) { 1711 PMD_DRV_LOG(INFO, 1712 "LLQ is not supported. Fallback to host mode policy.\n"); 1713 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1714 return 0; 1715 } 1716 1717 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 1718 if (unlikely(rc)) { 1719 PMD_INIT_LOG(WARNING, 1720 "Failed to config dev mode. Fallback to host mode policy.\n"); 1721 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1722 return 0; 1723 } 1724 1725 /* Nothing to config, exit */ 1726 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 1727 return 0; 1728 1729 if (!adapter->dev_mem_base) { 1730 PMD_DRV_LOG(ERR, 1731 "Unable to access LLQ BAR resource. Fallback to host mode policy.\n"); 1732 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1733 return 0; 1734 } 1735 1736 ena_dev->mem_bar = adapter->dev_mem_base; 1737 1738 return 0; 1739 } 1740 1741 static uint32_t ena_calc_max_io_queue_num(struct ena_com_dev *ena_dev, 1742 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1743 { 1744 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues; 1745 1746 /* Regular queues capabilities */ 1747 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 1748 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 1749 &get_feat_ctx->max_queue_ext.max_queue_ext; 1750 io_rx_num = RTE_MIN(max_queue_ext->max_rx_sq_num, 1751 max_queue_ext->max_rx_cq_num); 1752 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 1753 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 1754 } else { 1755 struct ena_admin_queue_feature_desc *max_queues = 1756 &get_feat_ctx->max_queues; 1757 io_tx_sq_num = max_queues->max_sq_num; 1758 io_tx_cq_num = max_queues->max_cq_num; 1759 io_rx_num = RTE_MIN(io_tx_sq_num, io_tx_cq_num); 1760 } 1761 1762 /* In case of LLQ use the llq number in the get feature cmd */ 1763 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 1764 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 1765 1766 max_num_io_queues = RTE_MIN(ENA_MAX_NUM_IO_QUEUES, io_rx_num); 1767 max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_sq_num); 1768 max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_cq_num); 1769 1770 if (unlikely(max_num_io_queues == 0)) { 1771 PMD_DRV_LOG(ERR, "Number of IO queues cannot not be 0\n"); 1772 return -EFAULT; 1773 } 1774 1775 return max_num_io_queues; 1776 } 1777 1778 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1779 { 1780 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 1781 struct rte_pci_device *pci_dev; 1782 struct rte_intr_handle *intr_handle; 1783 struct ena_adapter *adapter = eth_dev->data->dev_private; 1784 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1785 struct ena_com_dev_get_features_ctx get_feat_ctx; 1786 struct ena_llq_configurations llq_config; 1787 const char *queue_type_str; 1788 uint32_t max_num_io_queues; 1789 int rc; 1790 static int adapters_found; 1791 bool disable_meta_caching; 1792 bool wd_state = false; 1793 1794 eth_dev->dev_ops = &ena_dev_ops; 1795 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1796 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1797 eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; 1798 1799 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1800 return 0; 1801 1802 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 1803 1804 memset(adapter, 0, sizeof(struct ena_adapter)); 1805 ena_dev = &adapter->ena_dev; 1806 1807 adapter->edev_data = eth_dev->data; 1808 1809 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1810 1811 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", 1812 pci_dev->addr.domain, 1813 pci_dev->addr.bus, 1814 pci_dev->addr.devid, 1815 pci_dev->addr.function); 1816 1817 intr_handle = &pci_dev->intr_handle; 1818 1819 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1820 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1821 1822 if (!adapter->regs) { 1823 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", 1824 ENA_REGS_BAR); 1825 return -ENXIO; 1826 } 1827 1828 ena_dev->reg_bar = adapter->regs; 1829 /* This is a dummy pointer for ena_com functions. */ 1830 ena_dev->dmadev = adapter; 1831 1832 adapter->id_number = adapters_found; 1833 1834 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1835 adapter->id_number); 1836 1837 rc = ena_parse_devargs(adapter, pci_dev->device.devargs); 1838 if (rc != 0) { 1839 PMD_INIT_LOG(CRIT, "Failed to parse devargs\n"); 1840 goto err; 1841 } 1842 1843 /* device specific initialization routine */ 1844 rc = ena_device_init(ena_dev, pci_dev, &get_feat_ctx, &wd_state); 1845 if (rc) { 1846 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); 1847 goto err; 1848 } 1849 adapter->wd_state = wd_state; 1850 1851 set_default_llq_configurations(&llq_config, &get_feat_ctx.llq, 1852 adapter->use_large_llq_hdr); 1853 rc = ena_set_queues_placement_policy(adapter, ena_dev, 1854 &get_feat_ctx.llq, &llq_config); 1855 if (unlikely(rc)) { 1856 PMD_INIT_LOG(CRIT, "Failed to set placement policy\n"); 1857 return rc; 1858 } 1859 1860 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 1861 queue_type_str = "Regular"; 1862 else 1863 queue_type_str = "Low latency"; 1864 PMD_DRV_LOG(INFO, "Placement policy: %s\n", queue_type_str); 1865 1866 calc_queue_ctx.ena_dev = ena_dev; 1867 calc_queue_ctx.get_feat_ctx = &get_feat_ctx; 1868 1869 max_num_io_queues = ena_calc_max_io_queue_num(ena_dev, &get_feat_ctx); 1870 rc = ena_calc_io_queue_size(&calc_queue_ctx, 1871 adapter->use_large_llq_hdr); 1872 if (unlikely((rc != 0) || (max_num_io_queues == 0))) { 1873 rc = -EFAULT; 1874 goto err_device_destroy; 1875 } 1876 1877 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; 1878 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; 1879 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 1880 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 1881 adapter->max_num_io_queues = max_num_io_queues; 1882 1883 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1884 disable_meta_caching = 1885 !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags & 1886 BIT(ENA_ADMIN_DISABLE_META_CACHING)); 1887 } else { 1888 disable_meta_caching = false; 1889 } 1890 1891 /* prepare ring structures */ 1892 ena_init_rings(adapter, disable_meta_caching); 1893 1894 ena_config_debug_area(adapter); 1895 1896 /* Set max MTU for this device */ 1897 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1898 1899 /* set device support for offloads */ 1900 adapter->offloads.tso4_supported = (get_feat_ctx.offload.tx & 1901 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0; 1902 adapter->offloads.tx_csum_supported = (get_feat_ctx.offload.tx & 1903 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) != 0; 1904 adapter->offloads.rx_csum_supported = 1905 (get_feat_ctx.offload.rx_supported & 1906 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) != 0; 1907 1908 /* Copy MAC address and point DPDK to it */ 1909 eth_dev->data->mac_addrs = (struct rte_ether_addr *)adapter->mac_addr; 1910 rte_ether_addr_copy((struct rte_ether_addr *) 1911 get_feat_ctx.dev_attr.mac_addr, 1912 (struct rte_ether_addr *)adapter->mac_addr); 1913 1914 adapter->drv_stats = rte_zmalloc("adapter stats", 1915 sizeof(*adapter->drv_stats), 1916 RTE_CACHE_LINE_SIZE); 1917 if (!adapter->drv_stats) { 1918 PMD_DRV_LOG(ERR, 1919 "Failed to allocate memory for adapter statistics\n"); 1920 rc = -ENOMEM; 1921 goto err_delete_debug_area; 1922 } 1923 1924 rte_spinlock_init(&adapter->admin_lock); 1925 1926 rte_intr_callback_register(intr_handle, 1927 ena_interrupt_handler_rte, 1928 eth_dev); 1929 rte_intr_enable(intr_handle); 1930 ena_com_set_admin_polling_mode(ena_dev, false); 1931 ena_com_admin_aenq_enable(ena_dev); 1932 1933 if (adapters_found == 0) 1934 rte_timer_subsystem_init(); 1935 rte_timer_init(&adapter->timer_wd); 1936 1937 adapters_found++; 1938 adapter->state = ENA_ADAPTER_STATE_INIT; 1939 1940 return 0; 1941 1942 err_delete_debug_area: 1943 ena_com_delete_debug_area(ena_dev); 1944 1945 err_device_destroy: 1946 ena_com_delete_host_info(ena_dev); 1947 ena_com_admin_destroy(ena_dev); 1948 1949 err: 1950 return rc; 1951 } 1952 1953 static void ena_destroy_device(struct rte_eth_dev *eth_dev) 1954 { 1955 struct ena_adapter *adapter = eth_dev->data->dev_private; 1956 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1957 1958 if (adapter->state == ENA_ADAPTER_STATE_FREE) 1959 return; 1960 1961 ena_com_set_admin_running_state(ena_dev, false); 1962 1963 if (adapter->state != ENA_ADAPTER_STATE_CLOSED) 1964 ena_close(eth_dev); 1965 1966 ena_com_delete_debug_area(ena_dev); 1967 ena_com_delete_host_info(ena_dev); 1968 1969 ena_com_abort_admin_commands(ena_dev); 1970 ena_com_wait_for_abort_completion(ena_dev); 1971 ena_com_admin_destroy(ena_dev); 1972 ena_com_mmio_reg_read_request_destroy(ena_dev); 1973 1974 adapter->state = ENA_ADAPTER_STATE_FREE; 1975 } 1976 1977 static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev) 1978 { 1979 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1980 return 0; 1981 1982 ena_destroy_device(eth_dev); 1983 1984 return 0; 1985 } 1986 1987 static int ena_dev_configure(struct rte_eth_dev *dev) 1988 { 1989 struct ena_adapter *adapter = dev->data->dev_private; 1990 1991 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1992 1993 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 1994 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1995 1996 adapter->tx_selected_offloads = dev->data->dev_conf.txmode.offloads; 1997 adapter->rx_selected_offloads = dev->data->dev_conf.rxmode.offloads; 1998 return 0; 1999 } 2000 2001 static void ena_init_rings(struct ena_adapter *adapter, 2002 bool disable_meta_caching) 2003 { 2004 size_t i; 2005 2006 for (i = 0; i < adapter->max_num_io_queues; i++) { 2007 struct ena_ring *ring = &adapter->tx_ring[i]; 2008 2009 ring->configured = 0; 2010 ring->type = ENA_RING_TYPE_TX; 2011 ring->adapter = adapter; 2012 ring->id = i; 2013 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 2014 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 2015 ring->sgl_size = adapter->max_tx_sgl_size; 2016 ring->disable_meta_caching = disable_meta_caching; 2017 } 2018 2019 for (i = 0; i < adapter->max_num_io_queues; i++) { 2020 struct ena_ring *ring = &adapter->rx_ring[i]; 2021 2022 ring->configured = 0; 2023 ring->type = ENA_RING_TYPE_RX; 2024 ring->adapter = adapter; 2025 ring->id = i; 2026 ring->sgl_size = adapter->max_rx_sgl_size; 2027 } 2028 } 2029 2030 static int ena_infos_get(struct rte_eth_dev *dev, 2031 struct rte_eth_dev_info *dev_info) 2032 { 2033 struct ena_adapter *adapter; 2034 struct ena_com_dev *ena_dev; 2035 uint64_t rx_feat = 0, tx_feat = 0; 2036 2037 ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); 2038 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); 2039 adapter = dev->data->dev_private; 2040 2041 ena_dev = &adapter->ena_dev; 2042 ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); 2043 2044 dev_info->speed_capa = 2045 ETH_LINK_SPEED_1G | 2046 ETH_LINK_SPEED_2_5G | 2047 ETH_LINK_SPEED_5G | 2048 ETH_LINK_SPEED_10G | 2049 ETH_LINK_SPEED_25G | 2050 ETH_LINK_SPEED_40G | 2051 ETH_LINK_SPEED_50G | 2052 ETH_LINK_SPEED_100G; 2053 2054 /* Set Tx & Rx features available for device */ 2055 if (adapter->offloads.tso4_supported) 2056 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 2057 2058 if (adapter->offloads.tx_csum_supported) 2059 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 2060 DEV_TX_OFFLOAD_UDP_CKSUM | 2061 DEV_TX_OFFLOAD_TCP_CKSUM; 2062 2063 if (adapter->offloads.rx_csum_supported) 2064 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 2065 DEV_RX_OFFLOAD_UDP_CKSUM | 2066 DEV_RX_OFFLOAD_TCP_CKSUM; 2067 2068 rx_feat |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2069 2070 /* Inform framework about available features */ 2071 dev_info->rx_offload_capa = rx_feat; 2072 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_RSS_HASH; 2073 dev_info->rx_queue_offload_capa = rx_feat; 2074 dev_info->tx_offload_capa = tx_feat; 2075 dev_info->tx_queue_offload_capa = tx_feat; 2076 2077 dev_info->flow_type_rss_offloads = ETH_RSS_IP | ETH_RSS_TCP | 2078 ETH_RSS_UDP; 2079 2080 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 2081 dev_info->max_rx_pktlen = adapter->max_mtu; 2082 dev_info->max_mac_addrs = 1; 2083 2084 dev_info->max_rx_queues = adapter->max_num_io_queues; 2085 dev_info->max_tx_queues = adapter->max_num_io_queues; 2086 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 2087 2088 adapter->tx_supported_offloads = tx_feat; 2089 adapter->rx_supported_offloads = rx_feat; 2090 2091 dev_info->rx_desc_lim.nb_max = adapter->max_rx_ring_size; 2092 dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC; 2093 dev_info->rx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2094 adapter->max_rx_sgl_size); 2095 dev_info->rx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2096 adapter->max_rx_sgl_size); 2097 2098 dev_info->tx_desc_lim.nb_max = adapter->max_tx_ring_size; 2099 dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC; 2100 dev_info->tx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2101 adapter->max_tx_sgl_size); 2102 dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2103 adapter->max_tx_sgl_size); 2104 2105 dev_info->default_rxportconf.ring_size = ENA_DEFAULT_RING_SIZE; 2106 dev_info->default_txportconf.ring_size = ENA_DEFAULT_RING_SIZE; 2107 2108 return 0; 2109 } 2110 2111 static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len) 2112 { 2113 mbuf->data_len = len; 2114 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2115 mbuf->refcnt = 1; 2116 mbuf->next = NULL; 2117 } 2118 2119 static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, 2120 struct ena_com_rx_buf_info *ena_bufs, 2121 uint32_t descs, 2122 uint16_t *next_to_clean, 2123 uint8_t offset) 2124 { 2125 struct rte_mbuf *mbuf; 2126 struct rte_mbuf *mbuf_head; 2127 struct ena_rx_buffer *rx_info; 2128 int rc; 2129 uint16_t ntc, len, req_id, buf = 0; 2130 2131 if (unlikely(descs == 0)) 2132 return NULL; 2133 2134 ntc = *next_to_clean; 2135 2136 len = ena_bufs[buf].len; 2137 req_id = ena_bufs[buf].req_id; 2138 2139 rx_info = &rx_ring->rx_buffer_info[req_id]; 2140 2141 mbuf = rx_info->mbuf; 2142 RTE_ASSERT(mbuf != NULL); 2143 2144 ena_init_rx_mbuf(mbuf, len); 2145 2146 /* Fill the mbuf head with the data specific for 1st segment. */ 2147 mbuf_head = mbuf; 2148 mbuf_head->nb_segs = descs; 2149 mbuf_head->port = rx_ring->port_id; 2150 mbuf_head->pkt_len = len; 2151 mbuf_head->data_off += offset; 2152 2153 rx_info->mbuf = NULL; 2154 rx_ring->empty_rx_reqs[ntc] = req_id; 2155 ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); 2156 2157 while (--descs) { 2158 ++buf; 2159 len = ena_bufs[buf].len; 2160 req_id = ena_bufs[buf].req_id; 2161 2162 rx_info = &rx_ring->rx_buffer_info[req_id]; 2163 RTE_ASSERT(rx_info->mbuf != NULL); 2164 2165 if (unlikely(len == 0)) { 2166 /* 2167 * Some devices can pass descriptor with the length 0. 2168 * To avoid confusion, the PMD is simply putting the 2169 * descriptor back, as it was never used. We'll avoid 2170 * mbuf allocation that way. 2171 */ 2172 rc = ena_add_single_rx_desc(rx_ring->ena_com_io_sq, 2173 rx_info->mbuf, req_id); 2174 if (unlikely(rc != 0)) { 2175 /* Free the mbuf in case of an error. */ 2176 rte_mbuf_raw_free(rx_info->mbuf); 2177 } else { 2178 /* 2179 * If there was no error, just exit the loop as 2180 * 0 length descriptor is always the last one. 2181 */ 2182 break; 2183 } 2184 } else { 2185 /* Create an mbuf chain. */ 2186 mbuf->next = rx_info->mbuf; 2187 mbuf = mbuf->next; 2188 2189 ena_init_rx_mbuf(mbuf, len); 2190 mbuf_head->pkt_len += len; 2191 } 2192 2193 /* 2194 * Mark the descriptor as depleted and perform necessary 2195 * cleanup. 2196 * This code will execute in two cases: 2197 * 1. Descriptor len was greater than 0 - normal situation. 2198 * 2. Descriptor len was 0 and we failed to add the descriptor 2199 * to the device. In that situation, we should try to add 2200 * the mbuf again in the populate routine and mark the 2201 * descriptor as used up by the device. 2202 */ 2203 rx_info->mbuf = NULL; 2204 rx_ring->empty_rx_reqs[ntc] = req_id; 2205 ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); 2206 } 2207 2208 *next_to_clean = ntc; 2209 2210 return mbuf_head; 2211 } 2212 2213 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 2214 uint16_t nb_pkts) 2215 { 2216 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 2217 unsigned int free_queue_entries; 2218 unsigned int refill_threshold; 2219 uint16_t next_to_clean = rx_ring->next_to_clean; 2220 uint16_t descs_in_use; 2221 struct rte_mbuf *mbuf; 2222 uint16_t completed; 2223 struct ena_com_rx_ctx ena_rx_ctx; 2224 int i, rc = 0; 2225 2226 #ifdef RTE_ETHDEV_DEBUG_RX 2227 /* Check adapter state */ 2228 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2229 PMD_RX_LOG(ALERT, 2230 "Trying to receive pkts while device is NOT running\n"); 2231 return 0; 2232 } 2233 #endif 2234 2235 descs_in_use = rx_ring->ring_size - 2236 ena_com_free_q_entries(rx_ring->ena_com_io_sq) - 1; 2237 nb_pkts = RTE_MIN(descs_in_use, nb_pkts); 2238 2239 for (completed = 0; completed < nb_pkts; completed++) { 2240 ena_rx_ctx.max_bufs = rx_ring->sgl_size; 2241 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 2242 ena_rx_ctx.descs = 0; 2243 ena_rx_ctx.pkt_offset = 0; 2244 /* receive packet context */ 2245 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 2246 rx_ring->ena_com_io_sq, 2247 &ena_rx_ctx); 2248 if (unlikely(rc)) { 2249 PMD_RX_LOG(ERR, 2250 "Failed to get the packet from the device, rc: %d\n", 2251 rc); 2252 if (rc == ENA_COM_NO_SPACE) { 2253 ++rx_ring->rx_stats.bad_desc_num; 2254 rx_ring->adapter->reset_reason = 2255 ENA_REGS_RESET_TOO_MANY_RX_DESCS; 2256 } else { 2257 ++rx_ring->rx_stats.bad_req_id; 2258 rx_ring->adapter->reset_reason = 2259 ENA_REGS_RESET_INV_RX_REQ_ID; 2260 } 2261 rx_ring->adapter->trigger_reset = true; 2262 return 0; 2263 } 2264 2265 mbuf = ena_rx_mbuf(rx_ring, 2266 ena_rx_ctx.ena_bufs, 2267 ena_rx_ctx.descs, 2268 &next_to_clean, 2269 ena_rx_ctx.pkt_offset); 2270 if (unlikely(mbuf == NULL)) { 2271 for (i = 0; i < ena_rx_ctx.descs; ++i) { 2272 rx_ring->empty_rx_reqs[next_to_clean] = 2273 rx_ring->ena_bufs[i].req_id; 2274 next_to_clean = ENA_IDX_NEXT_MASKED( 2275 next_to_clean, rx_ring->size_mask); 2276 } 2277 break; 2278 } 2279 2280 /* fill mbuf attributes if any */ 2281 ena_rx_mbuf_prepare(mbuf, &ena_rx_ctx); 2282 2283 if (unlikely(mbuf->ol_flags & 2284 (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD))) { 2285 rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors); 2286 ++rx_ring->rx_stats.bad_csum; 2287 } 2288 2289 rx_pkts[completed] = mbuf; 2290 rx_ring->rx_stats.bytes += mbuf->pkt_len; 2291 } 2292 2293 rx_ring->rx_stats.cnt += completed; 2294 rx_ring->next_to_clean = next_to_clean; 2295 2296 free_queue_entries = ena_com_free_q_entries(rx_ring->ena_com_io_sq); 2297 refill_threshold = 2298 RTE_MIN(rx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, 2299 (unsigned int)ENA_REFILL_THRESH_PACKET); 2300 2301 /* Burst refill to save doorbells, memory barriers, const interval */ 2302 if (free_queue_entries > refill_threshold) { 2303 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); 2304 ena_populate_rx_queue(rx_ring, free_queue_entries); 2305 } 2306 2307 return completed; 2308 } 2309 2310 static uint16_t 2311 eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2312 uint16_t nb_pkts) 2313 { 2314 int32_t ret; 2315 uint32_t i; 2316 struct rte_mbuf *m; 2317 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2318 struct rte_ipv4_hdr *ip_hdr; 2319 uint64_t ol_flags; 2320 uint16_t frag_field; 2321 2322 for (i = 0; i != nb_pkts; i++) { 2323 m = tx_pkts[i]; 2324 ol_flags = m->ol_flags; 2325 2326 if (!(ol_flags & PKT_TX_IPV4)) 2327 continue; 2328 2329 /* If there was not L2 header length specified, assume it is 2330 * length of the ethernet header. 2331 */ 2332 if (unlikely(m->l2_len == 0)) 2333 m->l2_len = sizeof(struct rte_ether_hdr); 2334 2335 ip_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, 2336 m->l2_len); 2337 frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); 2338 2339 if ((frag_field & RTE_IPV4_HDR_DF_FLAG) != 0) { 2340 m->packet_type |= RTE_PTYPE_L4_NONFRAG; 2341 2342 /* If IPv4 header has DF flag enabled and TSO support is 2343 * disabled, partial chcecksum should not be calculated. 2344 */ 2345 if (!tx_ring->adapter->offloads.tso4_supported) 2346 continue; 2347 } 2348 2349 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || 2350 (ol_flags & PKT_TX_L4_MASK) == 2351 PKT_TX_SCTP_CKSUM) { 2352 rte_errno = ENOTSUP; 2353 return i; 2354 } 2355 2356 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 2357 ret = rte_validate_tx_offload(m); 2358 if (ret != 0) { 2359 rte_errno = -ret; 2360 return i; 2361 } 2362 #endif 2363 2364 /* In case we are supposed to TSO and have DF not set (DF=0) 2365 * hardware must be provided with partial checksum, otherwise 2366 * it will take care of necessary calculations. 2367 */ 2368 2369 ret = rte_net_intel_cksum_flags_prepare(m, 2370 ol_flags & ~PKT_TX_TCP_SEG); 2371 if (ret != 0) { 2372 rte_errno = -ret; 2373 return i; 2374 } 2375 } 2376 2377 return i; 2378 } 2379 2380 static void ena_update_hints(struct ena_adapter *adapter, 2381 struct ena_admin_ena_hw_hints *hints) 2382 { 2383 if (hints->admin_completion_tx_timeout) 2384 adapter->ena_dev.admin_queue.completion_timeout = 2385 hints->admin_completion_tx_timeout * 1000; 2386 2387 if (hints->mmio_read_timeout) 2388 /* convert to usec */ 2389 adapter->ena_dev.mmio_read.reg_read_to = 2390 hints->mmio_read_timeout * 1000; 2391 2392 if (hints->driver_watchdog_timeout) { 2393 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2394 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 2395 else 2396 // Convert msecs to ticks 2397 adapter->keep_alive_timeout = 2398 (hints->driver_watchdog_timeout * 2399 rte_get_timer_hz()) / 1000; 2400 } 2401 } 2402 2403 static int ena_check_space_and_linearize_mbuf(struct ena_ring *tx_ring, 2404 struct rte_mbuf *mbuf) 2405 { 2406 struct ena_com_dev *ena_dev; 2407 int num_segments, header_len, rc; 2408 2409 ena_dev = &tx_ring->adapter->ena_dev; 2410 num_segments = mbuf->nb_segs; 2411 header_len = mbuf->data_len; 2412 2413 if (likely(num_segments < tx_ring->sgl_size)) 2414 goto checkspace; 2415 2416 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && 2417 (num_segments == tx_ring->sgl_size) && 2418 (header_len < tx_ring->tx_max_header_size)) 2419 goto checkspace; 2420 2421 /* Checking for space for 2 additional metadata descriptors due to 2422 * possible header split and metadata descriptor. Linearization will 2423 * be needed so we reduce the segments number from num_segments to 1 2424 */ 2425 if (!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 3)) { 2426 PMD_TX_LOG(DEBUG, "Not enough space in the Tx queue\n"); 2427 return ENA_COM_NO_MEM; 2428 } 2429 ++tx_ring->tx_stats.linearize; 2430 rc = rte_pktmbuf_linearize(mbuf); 2431 if (unlikely(rc)) { 2432 PMD_TX_LOG(WARNING, "Mbuf linearize failed\n"); 2433 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 2434 ++tx_ring->tx_stats.linearize_failed; 2435 return rc; 2436 } 2437 2438 return 0; 2439 2440 checkspace: 2441 /* Checking for space for 2 additional metadata descriptors due to 2442 * possible header split and metadata descriptor 2443 */ 2444 if (!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 2445 num_segments + 2)) { 2446 PMD_TX_LOG(DEBUG, "Not enough space in the Tx queue\n"); 2447 return ENA_COM_NO_MEM; 2448 } 2449 2450 return 0; 2451 } 2452 2453 static void ena_tx_map_mbuf(struct ena_ring *tx_ring, 2454 struct ena_tx_buffer *tx_info, 2455 struct rte_mbuf *mbuf, 2456 void **push_header, 2457 uint16_t *header_len) 2458 { 2459 struct ena_com_buf *ena_buf; 2460 uint16_t delta, seg_len, push_len; 2461 2462 delta = 0; 2463 seg_len = mbuf->data_len; 2464 2465 tx_info->mbuf = mbuf; 2466 ena_buf = tx_info->bufs; 2467 2468 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2469 /* 2470 * Tx header might be (and will be in most cases) smaller than 2471 * tx_max_header_size. But it's not an issue to send more data 2472 * to the device, than actually needed if the mbuf size is 2473 * greater than tx_max_header_size. 2474 */ 2475 push_len = RTE_MIN(mbuf->pkt_len, tx_ring->tx_max_header_size); 2476 *header_len = push_len; 2477 2478 if (likely(push_len <= seg_len)) { 2479 /* If the push header is in the single segment, then 2480 * just point it to the 1st mbuf data. 2481 */ 2482 *push_header = rte_pktmbuf_mtod(mbuf, uint8_t *); 2483 } else { 2484 /* If the push header lays in the several segments, copy 2485 * it to the intermediate buffer. 2486 */ 2487 rte_pktmbuf_read(mbuf, 0, push_len, 2488 tx_ring->push_buf_intermediate_buf); 2489 *push_header = tx_ring->push_buf_intermediate_buf; 2490 delta = push_len - seg_len; 2491 } 2492 } else { 2493 *push_header = NULL; 2494 *header_len = 0; 2495 push_len = 0; 2496 } 2497 2498 /* Process first segment taking into consideration pushed header */ 2499 if (seg_len > push_len) { 2500 ena_buf->paddr = mbuf->buf_iova + 2501 mbuf->data_off + 2502 push_len; 2503 ena_buf->len = seg_len - push_len; 2504 ena_buf++; 2505 tx_info->num_of_bufs++; 2506 } 2507 2508 while ((mbuf = mbuf->next) != NULL) { 2509 seg_len = mbuf->data_len; 2510 2511 /* Skip mbufs if whole data is pushed as a header */ 2512 if (unlikely(delta > seg_len)) { 2513 delta -= seg_len; 2514 continue; 2515 } 2516 2517 ena_buf->paddr = mbuf->buf_iova + mbuf->data_off + delta; 2518 ena_buf->len = seg_len - delta; 2519 ena_buf++; 2520 tx_info->num_of_bufs++; 2521 2522 delta = 0; 2523 } 2524 } 2525 2526 static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf) 2527 { 2528 struct ena_tx_buffer *tx_info; 2529 struct ena_com_tx_ctx ena_tx_ctx = { { 0 } }; 2530 uint16_t next_to_use; 2531 uint16_t header_len; 2532 uint16_t req_id; 2533 void *push_header; 2534 int nb_hw_desc; 2535 int rc; 2536 2537 rc = ena_check_space_and_linearize_mbuf(tx_ring, mbuf); 2538 if (unlikely(rc)) 2539 return rc; 2540 2541 next_to_use = tx_ring->next_to_use; 2542 2543 req_id = tx_ring->empty_tx_reqs[next_to_use]; 2544 tx_info = &tx_ring->tx_buffer_info[req_id]; 2545 tx_info->num_of_bufs = 0; 2546 2547 ena_tx_map_mbuf(tx_ring, tx_info, mbuf, &push_header, &header_len); 2548 2549 ena_tx_ctx.ena_bufs = tx_info->bufs; 2550 ena_tx_ctx.push_header = push_header; 2551 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 2552 ena_tx_ctx.req_id = req_id; 2553 ena_tx_ctx.header_len = header_len; 2554 2555 /* Set Tx offloads flags, if applicable */ 2556 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads, 2557 tx_ring->disable_meta_caching); 2558 2559 if (unlikely(ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, 2560 &ena_tx_ctx))) { 2561 PMD_TX_LOG(DEBUG, 2562 "LLQ Tx max burst size of queue %d achieved, writing doorbell to send burst\n", 2563 tx_ring->id); 2564 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2565 tx_ring->tx_stats.doorbells++; 2566 tx_ring->pkts_without_db = false; 2567 } 2568 2569 /* prepare the packet's descriptors to dma engine */ 2570 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx, 2571 &nb_hw_desc); 2572 if (unlikely(rc)) { 2573 ++tx_ring->tx_stats.prepare_ctx_err; 2574 return rc; 2575 } 2576 2577 tx_info->tx_descs = nb_hw_desc; 2578 2579 tx_ring->tx_stats.cnt++; 2580 tx_ring->tx_stats.bytes += mbuf->pkt_len; 2581 2582 tx_ring->next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, 2583 tx_ring->size_mask); 2584 2585 return 0; 2586 } 2587 2588 static void ena_tx_cleanup(struct ena_ring *tx_ring) 2589 { 2590 unsigned int cleanup_budget; 2591 unsigned int total_tx_descs = 0; 2592 uint16_t next_to_clean = tx_ring->next_to_clean; 2593 2594 cleanup_budget = RTE_MIN(tx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, 2595 (unsigned int)ENA_REFILL_THRESH_PACKET); 2596 2597 while (likely(total_tx_descs < cleanup_budget)) { 2598 struct rte_mbuf *mbuf; 2599 struct ena_tx_buffer *tx_info; 2600 uint16_t req_id; 2601 2602 if (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) != 0) 2603 break; 2604 2605 if (unlikely(validate_tx_req_id(tx_ring, req_id) != 0)) 2606 break; 2607 2608 /* Get Tx info & store how many descs were processed */ 2609 tx_info = &tx_ring->tx_buffer_info[req_id]; 2610 2611 mbuf = tx_info->mbuf; 2612 rte_pktmbuf_free(mbuf); 2613 2614 tx_info->mbuf = NULL; 2615 tx_ring->empty_tx_reqs[next_to_clean] = req_id; 2616 2617 total_tx_descs += tx_info->tx_descs; 2618 2619 /* Put back descriptor to the ring for reuse */ 2620 next_to_clean = ENA_IDX_NEXT_MASKED(next_to_clean, 2621 tx_ring->size_mask); 2622 } 2623 2624 if (likely(total_tx_descs > 0)) { 2625 /* acknowledge completion of sent packets */ 2626 tx_ring->next_to_clean = next_to_clean; 2627 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 2628 ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); 2629 } 2630 } 2631 2632 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2633 uint16_t nb_pkts) 2634 { 2635 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2636 uint16_t sent_idx = 0; 2637 2638 #ifdef RTE_ETHDEV_DEBUG_TX 2639 /* Check adapter state */ 2640 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2641 PMD_TX_LOG(ALERT, 2642 "Trying to xmit pkts while device is NOT running\n"); 2643 return 0; 2644 } 2645 #endif 2646 2647 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 2648 if (ena_xmit_mbuf(tx_ring, tx_pkts[sent_idx])) 2649 break; 2650 tx_ring->pkts_without_db = true; 2651 rte_prefetch0(tx_pkts[ENA_IDX_ADD_MASKED(sent_idx, 4, 2652 tx_ring->size_mask)]); 2653 } 2654 2655 tx_ring->tx_stats.available_desc = 2656 ena_com_free_q_entries(tx_ring->ena_com_io_sq); 2657 2658 /* If there are ready packets to be xmitted... */ 2659 if (likely(tx_ring->pkts_without_db)) { 2660 /* ...let HW do its best :-) */ 2661 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2662 tx_ring->tx_stats.doorbells++; 2663 tx_ring->pkts_without_db = false; 2664 } 2665 2666 ena_tx_cleanup(tx_ring); 2667 2668 tx_ring->tx_stats.available_desc = 2669 ena_com_free_q_entries(tx_ring->ena_com_io_sq); 2670 tx_ring->tx_stats.tx_poll++; 2671 2672 return sent_idx; 2673 } 2674 2675 int ena_copy_eni_stats(struct ena_adapter *adapter) 2676 { 2677 struct ena_admin_eni_stats admin_eni_stats; 2678 int rc; 2679 2680 rte_spinlock_lock(&adapter->admin_lock); 2681 rc = ena_com_get_eni_stats(&adapter->ena_dev, &admin_eni_stats); 2682 rte_spinlock_unlock(&adapter->admin_lock); 2683 if (rc != 0) { 2684 if (rc == ENA_COM_UNSUPPORTED) { 2685 PMD_DRV_LOG(DEBUG, 2686 "Retrieving ENI metrics is not supported\n"); 2687 } else { 2688 PMD_DRV_LOG(WARNING, 2689 "Failed to get ENI metrics, rc: %d\n", rc); 2690 } 2691 return rc; 2692 } 2693 2694 rte_memcpy(&adapter->eni_stats, &admin_eni_stats, 2695 sizeof(struct ena_stats_eni)); 2696 2697 return 0; 2698 } 2699 2700 /** 2701 * DPDK callback to retrieve names of extended device statistics 2702 * 2703 * @param dev 2704 * Pointer to Ethernet device structure. 2705 * @param[out] xstats_names 2706 * Buffer to insert names into. 2707 * @param n 2708 * Number of names. 2709 * 2710 * @return 2711 * Number of xstats names. 2712 */ 2713 static int ena_xstats_get_names(struct rte_eth_dev *dev, 2714 struct rte_eth_xstat_name *xstats_names, 2715 unsigned int n) 2716 { 2717 unsigned int xstats_count = ena_xstats_calc_num(dev->data); 2718 unsigned int stat, i, count = 0; 2719 2720 if (n < xstats_count || !xstats_names) 2721 return xstats_count; 2722 2723 for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) 2724 strcpy(xstats_names[count].name, 2725 ena_stats_global_strings[stat].name); 2726 2727 for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++) 2728 strcpy(xstats_names[count].name, 2729 ena_stats_eni_strings[stat].name); 2730 2731 for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) 2732 for (i = 0; i < dev->data->nb_rx_queues; i++, count++) 2733 snprintf(xstats_names[count].name, 2734 sizeof(xstats_names[count].name), 2735 "rx_q%d_%s", i, 2736 ena_stats_rx_strings[stat].name); 2737 2738 for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) 2739 for (i = 0; i < dev->data->nb_tx_queues; i++, count++) 2740 snprintf(xstats_names[count].name, 2741 sizeof(xstats_names[count].name), 2742 "tx_q%d_%s", i, 2743 ena_stats_tx_strings[stat].name); 2744 2745 return xstats_count; 2746 } 2747 2748 /** 2749 * DPDK callback to get extended device statistics. 2750 * 2751 * @param dev 2752 * Pointer to Ethernet device structure. 2753 * @param[out] stats 2754 * Stats table output buffer. 2755 * @param n 2756 * The size of the stats table. 2757 * 2758 * @return 2759 * Number of xstats on success, negative on failure. 2760 */ 2761 static int ena_xstats_get(struct rte_eth_dev *dev, 2762 struct rte_eth_xstat *xstats, 2763 unsigned int n) 2764 { 2765 struct ena_adapter *adapter = dev->data->dev_private; 2766 unsigned int xstats_count = ena_xstats_calc_num(dev->data); 2767 unsigned int stat, i, count = 0; 2768 int stat_offset; 2769 void *stats_begin; 2770 2771 if (n < xstats_count) 2772 return xstats_count; 2773 2774 if (!xstats) 2775 return 0; 2776 2777 for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) { 2778 stat_offset = ena_stats_global_strings[stat].stat_offset; 2779 stats_begin = &adapter->dev_stats; 2780 2781 xstats[count].id = count; 2782 xstats[count].value = *((uint64_t *) 2783 ((char *)stats_begin + stat_offset)); 2784 } 2785 2786 /* Even if the function below fails, we should copy previous (or initial 2787 * values) to keep structure of rte_eth_xstat consistent. 2788 */ 2789 ena_copy_eni_stats(adapter); 2790 for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++) { 2791 stat_offset = ena_stats_eni_strings[stat].stat_offset; 2792 stats_begin = &adapter->eni_stats; 2793 2794 xstats[count].id = count; 2795 xstats[count].value = *((uint64_t *) 2796 ((char *)stats_begin + stat_offset)); 2797 } 2798 2799 for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) { 2800 for (i = 0; i < dev->data->nb_rx_queues; i++, count++) { 2801 stat_offset = ena_stats_rx_strings[stat].stat_offset; 2802 stats_begin = &adapter->rx_ring[i].rx_stats; 2803 2804 xstats[count].id = count; 2805 xstats[count].value = *((uint64_t *) 2806 ((char *)stats_begin + stat_offset)); 2807 } 2808 } 2809 2810 for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) { 2811 for (i = 0; i < dev->data->nb_tx_queues; i++, count++) { 2812 stat_offset = ena_stats_tx_strings[stat].stat_offset; 2813 stats_begin = &adapter->tx_ring[i].rx_stats; 2814 2815 xstats[count].id = count; 2816 xstats[count].value = *((uint64_t *) 2817 ((char *)stats_begin + stat_offset)); 2818 } 2819 } 2820 2821 return count; 2822 } 2823 2824 static int ena_xstats_get_by_id(struct rte_eth_dev *dev, 2825 const uint64_t *ids, 2826 uint64_t *values, 2827 unsigned int n) 2828 { 2829 struct ena_adapter *adapter = dev->data->dev_private; 2830 uint64_t id; 2831 uint64_t rx_entries, tx_entries; 2832 unsigned int i; 2833 int qid; 2834 int valid = 0; 2835 bool was_eni_copied = false; 2836 2837 for (i = 0; i < n; ++i) { 2838 id = ids[i]; 2839 /* Check if id belongs to global statistics */ 2840 if (id < ENA_STATS_ARRAY_GLOBAL) { 2841 values[i] = *((uint64_t *)&adapter->dev_stats + id); 2842 ++valid; 2843 continue; 2844 } 2845 2846 /* Check if id belongs to ENI statistics */ 2847 id -= ENA_STATS_ARRAY_GLOBAL; 2848 if (id < ENA_STATS_ARRAY_ENI) { 2849 /* Avoid reading ENI stats multiple times in a single 2850 * function call, as it requires communication with the 2851 * admin queue. 2852 */ 2853 if (!was_eni_copied) { 2854 was_eni_copied = true; 2855 ena_copy_eni_stats(adapter); 2856 } 2857 values[i] = *((uint64_t *)&adapter->eni_stats + id); 2858 ++valid; 2859 continue; 2860 } 2861 2862 /* Check if id belongs to rx queue statistics */ 2863 id -= ENA_STATS_ARRAY_ENI; 2864 rx_entries = ENA_STATS_ARRAY_RX * dev->data->nb_rx_queues; 2865 if (id < rx_entries) { 2866 qid = id % dev->data->nb_rx_queues; 2867 id /= dev->data->nb_rx_queues; 2868 values[i] = *((uint64_t *) 2869 &adapter->rx_ring[qid].rx_stats + id); 2870 ++valid; 2871 continue; 2872 } 2873 /* Check if id belongs to rx queue statistics */ 2874 id -= rx_entries; 2875 tx_entries = ENA_STATS_ARRAY_TX * dev->data->nb_tx_queues; 2876 if (id < tx_entries) { 2877 qid = id % dev->data->nb_tx_queues; 2878 id /= dev->data->nb_tx_queues; 2879 values[i] = *((uint64_t *) 2880 &adapter->tx_ring[qid].tx_stats + id); 2881 ++valid; 2882 continue; 2883 } 2884 } 2885 2886 return valid; 2887 } 2888 2889 static int ena_process_bool_devarg(const char *key, 2890 const char *value, 2891 void *opaque) 2892 { 2893 struct ena_adapter *adapter = opaque; 2894 bool bool_value; 2895 2896 /* Parse the value. */ 2897 if (strcmp(value, "1") == 0) { 2898 bool_value = true; 2899 } else if (strcmp(value, "0") == 0) { 2900 bool_value = false; 2901 } else { 2902 PMD_INIT_LOG(ERR, 2903 "Invalid value: '%s' for key '%s'. Accepted: '0' or '1'\n", 2904 value, key); 2905 return -EINVAL; 2906 } 2907 2908 /* Now, assign it to the proper adapter field. */ 2909 if (strcmp(key, ENA_DEVARG_LARGE_LLQ_HDR) == 0) 2910 adapter->use_large_llq_hdr = bool_value; 2911 2912 return 0; 2913 } 2914 2915 static int ena_parse_devargs(struct ena_adapter *adapter, 2916 struct rte_devargs *devargs) 2917 { 2918 static const char * const allowed_args[] = { 2919 ENA_DEVARG_LARGE_LLQ_HDR, 2920 NULL, 2921 }; 2922 struct rte_kvargs *kvlist; 2923 int rc; 2924 2925 if (devargs == NULL) 2926 return 0; 2927 2928 kvlist = rte_kvargs_parse(devargs->args, allowed_args); 2929 if (kvlist == NULL) { 2930 PMD_INIT_LOG(ERR, "Invalid device arguments: %s\n", 2931 devargs->args); 2932 return -EINVAL; 2933 } 2934 2935 rc = rte_kvargs_process(kvlist, ENA_DEVARG_LARGE_LLQ_HDR, 2936 ena_process_bool_devarg, adapter); 2937 2938 rte_kvargs_free(kvlist); 2939 2940 return rc; 2941 } 2942 2943 /********************************************************************* 2944 * PMD configuration 2945 *********************************************************************/ 2946 static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2947 struct rte_pci_device *pci_dev) 2948 { 2949 return rte_eth_dev_pci_generic_probe(pci_dev, 2950 sizeof(struct ena_adapter), eth_ena_dev_init); 2951 } 2952 2953 static int eth_ena_pci_remove(struct rte_pci_device *pci_dev) 2954 { 2955 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ena_dev_uninit); 2956 } 2957 2958 static struct rte_pci_driver rte_ena_pmd = { 2959 .id_table = pci_id_ena_map, 2960 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 2961 RTE_PCI_DRV_WC_ACTIVATE, 2962 .probe = eth_ena_pci_probe, 2963 .remove = eth_ena_pci_remove, 2964 }; 2965 2966 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd); 2967 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 2968 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci"); 2969 RTE_PMD_REGISTER_PARAM_STRING(net_ena, ENA_DEVARG_LARGE_LLQ_HDR "=<0|1>"); 2970 RTE_LOG_REGISTER_SUFFIX(ena_logtype_init, init, NOTICE); 2971 RTE_LOG_REGISTER_SUFFIX(ena_logtype_driver, driver, NOTICE); 2972 #ifdef RTE_ETHDEV_DEBUG_RX 2973 RTE_LOG_REGISTER_SUFFIX(ena_logtype_rx, rx, DEBUG); 2974 #endif 2975 #ifdef RTE_ETHDEV_DEBUG_TX 2976 RTE_LOG_REGISTER_SUFFIX(ena_logtype_tx, tx, DEBUG); 2977 #endif 2978 RTE_LOG_REGISTER_SUFFIX(ena_logtype_com, com, WARNING); 2979 2980 /****************************************************************************** 2981 ******************************** AENQ Handlers ******************************* 2982 *****************************************************************************/ 2983 static void ena_update_on_link_change(void *adapter_data, 2984 struct ena_admin_aenq_entry *aenq_e) 2985 { 2986 struct rte_eth_dev *eth_dev = adapter_data; 2987 struct ena_adapter *adapter = eth_dev->data->dev_private; 2988 struct ena_admin_aenq_link_change_desc *aenq_link_desc; 2989 uint32_t status; 2990 2991 aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 2992 2993 status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc); 2994 adapter->link_status = status; 2995 2996 ena_link_update(eth_dev, 0); 2997 rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2998 } 2999 3000 static void ena_notification(void *adapter_data, 3001 struct ena_admin_aenq_entry *aenq_e) 3002 { 3003 struct rte_eth_dev *eth_dev = adapter_data; 3004 struct ena_adapter *adapter = eth_dev->data->dev_private; 3005 struct ena_admin_ena_hw_hints *hints; 3006 3007 if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION) 3008 PMD_DRV_LOG(WARNING, "Invalid AENQ group: %x. Expected: %x\n", 3009 aenq_e->aenq_common_desc.group, 3010 ENA_ADMIN_NOTIFICATION); 3011 3012 switch (aenq_e->aenq_common_desc.syndrome) { 3013 case ENA_ADMIN_UPDATE_HINTS: 3014 hints = (struct ena_admin_ena_hw_hints *) 3015 (&aenq_e->inline_data_w4); 3016 ena_update_hints(adapter, hints); 3017 break; 3018 default: 3019 PMD_DRV_LOG(ERR, "Invalid AENQ notification link state: %d\n", 3020 aenq_e->aenq_common_desc.syndrome); 3021 } 3022 } 3023 3024 static void ena_keep_alive(void *adapter_data, 3025 __rte_unused struct ena_admin_aenq_entry *aenq_e) 3026 { 3027 struct rte_eth_dev *eth_dev = adapter_data; 3028 struct ena_adapter *adapter = eth_dev->data->dev_private; 3029 struct ena_admin_aenq_keep_alive_desc *desc; 3030 uint64_t rx_drops; 3031 uint64_t tx_drops; 3032 3033 adapter->timestamp_wd = rte_get_timer_cycles(); 3034 3035 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 3036 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 3037 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low; 3038 3039 adapter->drv_stats->rx_drops = rx_drops; 3040 adapter->dev_stats.tx_drops = tx_drops; 3041 } 3042 3043 /** 3044 * This handler will called for unknown event group or unimplemented handlers 3045 **/ 3046 static void unimplemented_aenq_handler(__rte_unused void *data, 3047 __rte_unused struct ena_admin_aenq_entry *aenq_e) 3048 { 3049 PMD_DRV_LOG(ERR, 3050 "Unknown event was received or event with unimplemented handler\n"); 3051 } 3052 3053 static struct ena_aenq_handlers aenq_handlers = { 3054 .handlers = { 3055 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 3056 [ENA_ADMIN_NOTIFICATION] = ena_notification, 3057 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive 3058 }, 3059 .unimplemented_handler = unimplemented_aenq_handler 3060 }; 3061