1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #include <rte_string_fns.h> 7 #include <rte_ether.h> 8 #include <ethdev_driver.h> 9 #include <ethdev_pci.h> 10 #include <rte_tcp.h> 11 #include <rte_atomic.h> 12 #include <rte_dev.h> 13 #include <rte_errno.h> 14 #include <rte_version.h> 15 #include <rte_net.h> 16 #include <rte_kvargs.h> 17 18 #include "ena_ethdev.h" 19 #include "ena_logs.h" 20 #include "ena_platform.h" 21 #include "ena_com.h" 22 #include "ena_eth_com.h" 23 24 #include <ena_common_defs.h> 25 #include <ena_regs_defs.h> 26 #include <ena_admin_defs.h> 27 #include <ena_eth_io_defs.h> 28 29 #define DRV_MODULE_VER_MAJOR 2 30 #define DRV_MODULE_VER_MINOR 3 31 #define DRV_MODULE_VER_SUBMINOR 0 32 33 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 34 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 35 /*reverse version of ENA_IO_RXQ_IDX*/ 36 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 37 38 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 39 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 40 41 #define GET_L4_HDR_LEN(mbuf) \ 42 ((rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, \ 43 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 44 45 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 46 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 47 #define ENA_HASH_KEY_SIZE 40 48 #define ETH_GSTRING_LEN 32 49 50 #define ARRAY_SIZE(x) RTE_DIM(x) 51 52 #define ENA_MIN_RING_DESC 128 53 54 #define ENA_PTYPE_HAS_HASH (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP) 55 56 enum ethtool_stringset { 57 ETH_SS_TEST = 0, 58 ETH_SS_STATS, 59 }; 60 61 struct ena_stats { 62 char name[ETH_GSTRING_LEN]; 63 int stat_offset; 64 }; 65 66 #define ENA_STAT_ENTRY(stat, stat_type) { \ 67 .name = #stat, \ 68 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 69 } 70 71 #define ENA_STAT_RX_ENTRY(stat) \ 72 ENA_STAT_ENTRY(stat, rx) 73 74 #define ENA_STAT_TX_ENTRY(stat) \ 75 ENA_STAT_ENTRY(stat, tx) 76 77 #define ENA_STAT_ENI_ENTRY(stat) \ 78 ENA_STAT_ENTRY(stat, eni) 79 80 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 81 ENA_STAT_ENTRY(stat, dev) 82 83 /* Device arguments */ 84 #define ENA_DEVARG_LARGE_LLQ_HDR "large_llq_hdr" 85 86 /* 87 * Each rte_memzone should have unique name. 88 * To satisfy it, count number of allocation and add it to name. 89 */ 90 rte_atomic64_t ena_alloc_cnt; 91 92 static const struct ena_stats ena_stats_global_strings[] = { 93 ENA_STAT_GLOBAL_ENTRY(wd_expired), 94 ENA_STAT_GLOBAL_ENTRY(dev_start), 95 ENA_STAT_GLOBAL_ENTRY(dev_stop), 96 ENA_STAT_GLOBAL_ENTRY(tx_drops), 97 }; 98 99 static const struct ena_stats ena_stats_eni_strings[] = { 100 ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded), 101 ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded), 102 ENA_STAT_ENI_ENTRY(pps_allowance_exceeded), 103 ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded), 104 ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded), 105 }; 106 107 static const struct ena_stats ena_stats_tx_strings[] = { 108 ENA_STAT_TX_ENTRY(cnt), 109 ENA_STAT_TX_ENTRY(bytes), 110 ENA_STAT_TX_ENTRY(prepare_ctx_err), 111 ENA_STAT_TX_ENTRY(linearize), 112 ENA_STAT_TX_ENTRY(linearize_failed), 113 ENA_STAT_TX_ENTRY(tx_poll), 114 ENA_STAT_TX_ENTRY(doorbells), 115 ENA_STAT_TX_ENTRY(bad_req_id), 116 ENA_STAT_TX_ENTRY(available_desc), 117 }; 118 119 static const struct ena_stats ena_stats_rx_strings[] = { 120 ENA_STAT_RX_ENTRY(cnt), 121 ENA_STAT_RX_ENTRY(bytes), 122 ENA_STAT_RX_ENTRY(refill_partial), 123 ENA_STAT_RX_ENTRY(bad_csum), 124 ENA_STAT_RX_ENTRY(mbuf_alloc_fail), 125 ENA_STAT_RX_ENTRY(bad_desc_num), 126 ENA_STAT_RX_ENTRY(bad_req_id), 127 }; 128 129 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 130 #define ENA_STATS_ARRAY_ENI ARRAY_SIZE(ena_stats_eni_strings) 131 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 132 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 133 134 #define QUEUE_OFFLOADS (DEV_TX_OFFLOAD_TCP_CKSUM |\ 135 DEV_TX_OFFLOAD_UDP_CKSUM |\ 136 DEV_TX_OFFLOAD_IPV4_CKSUM |\ 137 DEV_TX_OFFLOAD_TCP_TSO) 138 #define MBUF_OFFLOADS (PKT_TX_L4_MASK |\ 139 PKT_TX_IP_CKSUM |\ 140 PKT_TX_TCP_SEG) 141 142 /** Vendor ID used by Amazon devices */ 143 #define PCI_VENDOR_ID_AMAZON 0x1D0F 144 /** Amazon devices */ 145 #define PCI_DEVICE_ID_ENA_VF 0xEC20 146 #define PCI_DEVICE_ID_ENA_VF_RSERV0 0xEC21 147 148 #define ENA_TX_OFFLOAD_MASK (\ 149 PKT_TX_L4_MASK | \ 150 PKT_TX_IPV6 | \ 151 PKT_TX_IPV4 | \ 152 PKT_TX_IP_CKSUM | \ 153 PKT_TX_TCP_SEG) 154 155 #define ENA_TX_OFFLOAD_NOTSUP_MASK \ 156 (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) 157 158 static const struct rte_pci_id pci_id_ena_map[] = { 159 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 160 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF_RSERV0) }, 161 { .device_id = 0 }, 162 }; 163 164 static struct ena_aenq_handlers aenq_handlers; 165 166 static int ena_device_init(struct ena_com_dev *ena_dev, 167 struct rte_pci_device *pdev, 168 struct ena_com_dev_get_features_ctx *get_feat_ctx, 169 bool *wd_state); 170 static int ena_dev_configure(struct rte_eth_dev *dev); 171 static void ena_tx_map_mbuf(struct ena_ring *tx_ring, 172 struct ena_tx_buffer *tx_info, 173 struct rte_mbuf *mbuf, 174 void **push_header, 175 uint16_t *header_len); 176 static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf); 177 static void ena_tx_cleanup(struct ena_ring *tx_ring); 178 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 179 uint16_t nb_pkts); 180 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 181 uint16_t nb_pkts); 182 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 183 uint16_t nb_desc, unsigned int socket_id, 184 const struct rte_eth_txconf *tx_conf); 185 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 186 uint16_t nb_desc, unsigned int socket_id, 187 const struct rte_eth_rxconf *rx_conf, 188 struct rte_mempool *mp); 189 static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len); 190 static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, 191 struct ena_com_rx_buf_info *ena_bufs, 192 uint32_t descs, 193 uint16_t *next_to_clean, 194 uint8_t offset); 195 static uint16_t eth_ena_recv_pkts(void *rx_queue, 196 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 197 static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, 198 struct rte_mbuf *mbuf, uint16_t id); 199 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 200 static void ena_init_rings(struct ena_adapter *adapter, 201 bool disable_meta_caching); 202 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 203 static int ena_start(struct rte_eth_dev *dev); 204 static int ena_stop(struct rte_eth_dev *dev); 205 static int ena_close(struct rte_eth_dev *dev); 206 static int ena_dev_reset(struct rte_eth_dev *dev); 207 static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 208 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 209 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 210 static void ena_rx_queue_release(void *queue); 211 static void ena_tx_queue_release(void *queue); 212 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 213 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 214 static int ena_link_update(struct rte_eth_dev *dev, 215 int wait_to_complete); 216 static int ena_create_io_queue(struct ena_ring *ring); 217 static void ena_queue_stop(struct ena_ring *ring); 218 static void ena_queue_stop_all(struct rte_eth_dev *dev, 219 enum ena_ring_type ring_type); 220 static int ena_queue_start(struct ena_ring *ring); 221 static int ena_queue_start_all(struct rte_eth_dev *dev, 222 enum ena_ring_type ring_type); 223 static void ena_stats_restart(struct rte_eth_dev *dev); 224 static int ena_infos_get(struct rte_eth_dev *dev, 225 struct rte_eth_dev_info *dev_info); 226 static int ena_rss_reta_update(struct rte_eth_dev *dev, 227 struct rte_eth_rss_reta_entry64 *reta_conf, 228 uint16_t reta_size); 229 static int ena_rss_reta_query(struct rte_eth_dev *dev, 230 struct rte_eth_rss_reta_entry64 *reta_conf, 231 uint16_t reta_size); 232 static void ena_interrupt_handler_rte(void *cb_arg); 233 static void ena_timer_wd_callback(struct rte_timer *timer, void *arg); 234 static void ena_destroy_device(struct rte_eth_dev *eth_dev); 235 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev); 236 static int ena_xstats_get_names(struct rte_eth_dev *dev, 237 struct rte_eth_xstat_name *xstats_names, 238 unsigned int n); 239 static int ena_xstats_get(struct rte_eth_dev *dev, 240 struct rte_eth_xstat *stats, 241 unsigned int n); 242 static int ena_xstats_get_by_id(struct rte_eth_dev *dev, 243 const uint64_t *ids, 244 uint64_t *values, 245 unsigned int n); 246 static int ena_process_bool_devarg(const char *key, 247 const char *value, 248 void *opaque); 249 static int ena_parse_devargs(struct ena_adapter *adapter, 250 struct rte_devargs *devargs); 251 static int ena_copy_eni_stats(struct ena_adapter *adapter); 252 253 static const struct eth_dev_ops ena_dev_ops = { 254 .dev_configure = ena_dev_configure, 255 .dev_infos_get = ena_infos_get, 256 .rx_queue_setup = ena_rx_queue_setup, 257 .tx_queue_setup = ena_tx_queue_setup, 258 .dev_start = ena_start, 259 .dev_stop = ena_stop, 260 .link_update = ena_link_update, 261 .stats_get = ena_stats_get, 262 .xstats_get_names = ena_xstats_get_names, 263 .xstats_get = ena_xstats_get, 264 .xstats_get_by_id = ena_xstats_get_by_id, 265 .mtu_set = ena_mtu_set, 266 .rx_queue_release = ena_rx_queue_release, 267 .tx_queue_release = ena_tx_queue_release, 268 .dev_close = ena_close, 269 .dev_reset = ena_dev_reset, 270 .reta_update = ena_rss_reta_update, 271 .reta_query = ena_rss_reta_query, 272 }; 273 274 void ena_rss_key_fill(void *key, size_t size) 275 { 276 static bool key_generated; 277 static uint8_t default_key[ENA_HASH_KEY_SIZE]; 278 size_t i; 279 280 RTE_ASSERT(size <= ENA_HASH_KEY_SIZE); 281 282 if (!key_generated) { 283 for (i = 0; i < ENA_HASH_KEY_SIZE; ++i) 284 default_key[i] = rte_rand() & 0xff; 285 key_generated = true; 286 } 287 288 rte_memcpy(key, default_key, size); 289 } 290 291 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 292 struct ena_com_rx_ctx *ena_rx_ctx) 293 { 294 uint64_t ol_flags = 0; 295 uint32_t packet_type = 0; 296 297 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 298 packet_type |= RTE_PTYPE_L4_TCP; 299 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 300 packet_type |= RTE_PTYPE_L4_UDP; 301 302 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) { 303 packet_type |= RTE_PTYPE_L3_IPV4; 304 if (unlikely(ena_rx_ctx->l3_csum_err)) 305 ol_flags |= PKT_RX_IP_CKSUM_BAD; 306 else 307 ol_flags |= PKT_RX_IP_CKSUM_GOOD; 308 } else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) { 309 packet_type |= RTE_PTYPE_L3_IPV6; 310 } 311 312 if (!ena_rx_ctx->l4_csum_checked || ena_rx_ctx->frag) 313 ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 314 else 315 if (unlikely(ena_rx_ctx->l4_csum_err)) 316 ol_flags |= PKT_RX_L4_CKSUM_BAD; 317 else 318 ol_flags |= PKT_RX_L4_CKSUM_GOOD; 319 320 if (likely((packet_type & ENA_PTYPE_HAS_HASH) && !ena_rx_ctx->frag)) { 321 ol_flags |= PKT_RX_RSS_HASH; 322 mbuf->hash.rss = ena_rx_ctx->hash; 323 } 324 325 mbuf->ol_flags = ol_flags; 326 mbuf->packet_type = packet_type; 327 } 328 329 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 330 struct ena_com_tx_ctx *ena_tx_ctx, 331 uint64_t queue_offloads, 332 bool disable_meta_caching) 333 { 334 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 335 336 if ((mbuf->ol_flags & MBUF_OFFLOADS) && 337 (queue_offloads & QUEUE_OFFLOADS)) { 338 /* check if TSO is required */ 339 if ((mbuf->ol_flags & PKT_TX_TCP_SEG) && 340 (queue_offloads & DEV_TX_OFFLOAD_TCP_TSO)) { 341 ena_tx_ctx->tso_enable = true; 342 343 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 344 } 345 346 /* check if L3 checksum is needed */ 347 if ((mbuf->ol_flags & PKT_TX_IP_CKSUM) && 348 (queue_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)) 349 ena_tx_ctx->l3_csum_enable = true; 350 351 if (mbuf->ol_flags & PKT_TX_IPV6) { 352 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 353 } else { 354 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 355 356 /* set don't fragment (DF) flag */ 357 if (mbuf->packet_type & 358 (RTE_PTYPE_L4_NONFRAG 359 | RTE_PTYPE_INNER_L4_NONFRAG)) 360 ena_tx_ctx->df = true; 361 } 362 363 /* check if L4 checksum is needed */ 364 if (((mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) && 365 (queue_offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) { 366 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 367 ena_tx_ctx->l4_csum_enable = true; 368 } else if (((mbuf->ol_flags & PKT_TX_L4_MASK) == 369 PKT_TX_UDP_CKSUM) && 370 (queue_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) { 371 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 372 ena_tx_ctx->l4_csum_enable = true; 373 } else { 374 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 375 ena_tx_ctx->l4_csum_enable = false; 376 } 377 378 ena_meta->mss = mbuf->tso_segsz; 379 ena_meta->l3_hdr_len = mbuf->l3_len; 380 ena_meta->l3_hdr_offset = mbuf->l2_len; 381 382 ena_tx_ctx->meta_valid = true; 383 } else if (disable_meta_caching) { 384 memset(ena_meta, 0, sizeof(*ena_meta)); 385 ena_tx_ctx->meta_valid = true; 386 } else { 387 ena_tx_ctx->meta_valid = false; 388 } 389 } 390 391 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) 392 { 393 struct ena_tx_buffer *tx_info = NULL; 394 395 if (likely(req_id < tx_ring->ring_size)) { 396 tx_info = &tx_ring->tx_buffer_info[req_id]; 397 if (likely(tx_info->mbuf)) 398 return 0; 399 } 400 401 if (tx_info) 402 PMD_DRV_LOG(ERR, "tx_info doesn't have valid mbuf\n"); 403 else 404 PMD_DRV_LOG(ERR, "Invalid req_id: %hu\n", req_id); 405 406 /* Trigger device reset */ 407 ++tx_ring->tx_stats.bad_req_id; 408 tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; 409 tx_ring->adapter->trigger_reset = true; 410 return -EFAULT; 411 } 412 413 static void ena_config_host_info(struct ena_com_dev *ena_dev) 414 { 415 struct ena_admin_host_info *host_info; 416 int rc; 417 418 /* Allocate only the host info */ 419 rc = ena_com_allocate_host_info(ena_dev); 420 if (rc) { 421 PMD_DRV_LOG(ERR, "Cannot allocate host info\n"); 422 return; 423 } 424 425 host_info = ena_dev->host_attr.host_info; 426 427 host_info->os_type = ENA_ADMIN_OS_DPDK; 428 host_info->kernel_ver = RTE_VERSION; 429 strlcpy((char *)host_info->kernel_ver_str, rte_version(), 430 sizeof(host_info->kernel_ver_str)); 431 host_info->os_dist = RTE_VERSION; 432 strlcpy((char *)host_info->os_dist_str, rte_version(), 433 sizeof(host_info->os_dist_str)); 434 host_info->driver_version = 435 (DRV_MODULE_VER_MAJOR) | 436 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 437 (DRV_MODULE_VER_SUBMINOR << 438 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 439 host_info->num_cpus = rte_lcore_count(); 440 441 host_info->driver_supported_features = 442 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK; 443 444 rc = ena_com_set_host_attributes(ena_dev); 445 if (rc) { 446 if (rc == -ENA_COM_UNSUPPORTED) 447 PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); 448 else 449 PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); 450 451 goto err; 452 } 453 454 return; 455 456 err: 457 ena_com_delete_host_info(ena_dev); 458 } 459 460 /* This function calculates the number of xstats based on the current config */ 461 static unsigned int ena_xstats_calc_num(struct rte_eth_dev_data *data) 462 { 463 return ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENI + 464 (data->nb_tx_queues * ENA_STATS_ARRAY_TX) + 465 (data->nb_rx_queues * ENA_STATS_ARRAY_RX); 466 } 467 468 static void ena_config_debug_area(struct ena_adapter *adapter) 469 { 470 u32 debug_area_size; 471 int rc, ss_count; 472 473 ss_count = ena_xstats_calc_num(adapter->edev_data); 474 475 /* allocate 32 bytes for each string and 64bit for the value */ 476 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 477 478 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 479 if (rc) { 480 PMD_DRV_LOG(ERR, "Cannot allocate debug area\n"); 481 return; 482 } 483 484 rc = ena_com_set_host_attributes(&adapter->ena_dev); 485 if (rc) { 486 if (rc == -ENA_COM_UNSUPPORTED) 487 PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); 488 else 489 PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); 490 491 goto err; 492 } 493 494 return; 495 err: 496 ena_com_delete_debug_area(&adapter->ena_dev); 497 } 498 499 static int ena_close(struct rte_eth_dev *dev) 500 { 501 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 502 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 503 struct ena_adapter *adapter = dev->data->dev_private; 504 int ret = 0; 505 506 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 507 return 0; 508 509 if (adapter->state == ENA_ADAPTER_STATE_RUNNING) 510 ret = ena_stop(dev); 511 adapter->state = ENA_ADAPTER_STATE_CLOSED; 512 513 ena_rx_queue_release_all(dev); 514 ena_tx_queue_release_all(dev); 515 516 rte_free(adapter->drv_stats); 517 adapter->drv_stats = NULL; 518 519 rte_intr_disable(intr_handle); 520 rte_intr_callback_unregister(intr_handle, 521 ena_interrupt_handler_rte, 522 dev); 523 524 /* 525 * MAC is not allocated dynamically. Setting NULL should prevent from 526 * release of the resource in the rte_eth_dev_release_port(). 527 */ 528 dev->data->mac_addrs = NULL; 529 530 return ret; 531 } 532 533 static int 534 ena_dev_reset(struct rte_eth_dev *dev) 535 { 536 int rc = 0; 537 538 /* Cannot release memory in secondary process */ 539 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 540 PMD_DRV_LOG(WARNING, "dev_reset not supported in secondary.\n"); 541 return -EPERM; 542 } 543 544 ena_destroy_device(dev); 545 rc = eth_ena_dev_init(dev); 546 if (rc) 547 PMD_INIT_LOG(CRIT, "Cannot initialize device"); 548 549 return rc; 550 } 551 552 static int ena_rss_reta_update(struct rte_eth_dev *dev, 553 struct rte_eth_rss_reta_entry64 *reta_conf, 554 uint16_t reta_size) 555 { 556 struct ena_adapter *adapter = dev->data->dev_private; 557 struct ena_com_dev *ena_dev = &adapter->ena_dev; 558 int rc, i; 559 u16 entry_value; 560 int conf_idx; 561 int idx; 562 563 if ((reta_size == 0) || (reta_conf == NULL)) 564 return -EINVAL; 565 566 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 567 PMD_DRV_LOG(WARNING, 568 "indirection table %d is bigger than supported (%d)\n", 569 reta_size, ENA_RX_RSS_TABLE_SIZE); 570 return -EINVAL; 571 } 572 573 for (i = 0 ; i < reta_size ; i++) { 574 /* each reta_conf is for 64 entries. 575 * to support 128 we use 2 conf of 64 576 */ 577 conf_idx = i / RTE_RETA_GROUP_SIZE; 578 idx = i % RTE_RETA_GROUP_SIZE; 579 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 580 entry_value = 581 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 582 583 rc = ena_com_indirect_table_fill_entry(ena_dev, 584 i, 585 entry_value); 586 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 587 PMD_DRV_LOG(ERR, 588 "Cannot fill indirect table\n"); 589 return rc; 590 } 591 } 592 } 593 594 rte_spinlock_lock(&adapter->admin_lock); 595 rc = ena_com_indirect_table_set(ena_dev); 596 rte_spinlock_unlock(&adapter->admin_lock); 597 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 598 PMD_DRV_LOG(ERR, "Cannot flush the indirect table\n"); 599 return rc; 600 } 601 602 PMD_DRV_LOG(DEBUG, "%s(): RSS configured %d entries for port %d\n", 603 __func__, reta_size, dev->data->port_id); 604 605 return 0; 606 } 607 608 /* Query redirection table. */ 609 static int ena_rss_reta_query(struct rte_eth_dev *dev, 610 struct rte_eth_rss_reta_entry64 *reta_conf, 611 uint16_t reta_size) 612 { 613 struct ena_adapter *adapter = dev->data->dev_private; 614 struct ena_com_dev *ena_dev = &adapter->ena_dev; 615 int rc; 616 int i; 617 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 618 int reta_conf_idx; 619 int reta_idx; 620 621 if (reta_size == 0 || reta_conf == NULL || 622 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 623 return -EINVAL; 624 625 rte_spinlock_lock(&adapter->admin_lock); 626 rc = ena_com_indirect_table_get(ena_dev, indirect_table); 627 rte_spinlock_unlock(&adapter->admin_lock); 628 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 629 PMD_DRV_LOG(ERR, "cannot get indirect table\n"); 630 return -ENOTSUP; 631 } 632 633 for (i = 0 ; i < reta_size ; i++) { 634 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 635 reta_idx = i % RTE_RETA_GROUP_SIZE; 636 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 637 reta_conf[reta_conf_idx].reta[reta_idx] = 638 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 639 } 640 641 return 0; 642 } 643 644 static int ena_rss_init_default(struct ena_adapter *adapter) 645 { 646 struct ena_com_dev *ena_dev = &adapter->ena_dev; 647 uint16_t nb_rx_queues = adapter->edev_data->nb_rx_queues; 648 int rc, i; 649 u32 val; 650 651 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 652 if (unlikely(rc)) { 653 PMD_DRV_LOG(ERR, "Cannot init indirect table\n"); 654 goto err_rss_init; 655 } 656 657 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 658 val = i % nb_rx_queues; 659 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 660 ENA_IO_RXQ_IDX(val)); 661 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 662 PMD_DRV_LOG(ERR, "Cannot fill indirect table\n"); 663 goto err_fill_indir; 664 } 665 } 666 667 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 668 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 669 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 670 PMD_DRV_LOG(INFO, "Cannot fill hash function\n"); 671 goto err_fill_indir; 672 } 673 674 rc = ena_com_set_default_hash_ctrl(ena_dev); 675 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 676 PMD_DRV_LOG(INFO, "Cannot fill hash control\n"); 677 goto err_fill_indir; 678 } 679 680 rc = ena_com_indirect_table_set(ena_dev); 681 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 682 PMD_DRV_LOG(ERR, "Cannot flush the indirect table\n"); 683 goto err_fill_indir; 684 } 685 PMD_DRV_LOG(DEBUG, "RSS configured for port %d\n", 686 adapter->edev_data->port_id); 687 688 return 0; 689 690 err_fill_indir: 691 ena_com_rss_destroy(ena_dev); 692 err_rss_init: 693 694 return rc; 695 } 696 697 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 698 { 699 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 700 int nb_queues = dev->data->nb_rx_queues; 701 int i; 702 703 for (i = 0; i < nb_queues; i++) 704 ena_rx_queue_release(queues[i]); 705 } 706 707 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 708 { 709 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 710 int nb_queues = dev->data->nb_tx_queues; 711 int i; 712 713 for (i = 0; i < nb_queues; i++) 714 ena_tx_queue_release(queues[i]); 715 } 716 717 static void ena_rx_queue_release(void *queue) 718 { 719 struct ena_ring *ring = (struct ena_ring *)queue; 720 721 /* Free ring resources */ 722 if (ring->rx_buffer_info) 723 rte_free(ring->rx_buffer_info); 724 ring->rx_buffer_info = NULL; 725 726 if (ring->rx_refill_buffer) 727 rte_free(ring->rx_refill_buffer); 728 ring->rx_refill_buffer = NULL; 729 730 if (ring->empty_rx_reqs) 731 rte_free(ring->empty_rx_reqs); 732 ring->empty_rx_reqs = NULL; 733 734 ring->configured = 0; 735 736 PMD_DRV_LOG(NOTICE, "RX Queue %d:%d released\n", 737 ring->port_id, ring->id); 738 } 739 740 static void ena_tx_queue_release(void *queue) 741 { 742 struct ena_ring *ring = (struct ena_ring *)queue; 743 744 /* Free ring resources */ 745 if (ring->push_buf_intermediate_buf) 746 rte_free(ring->push_buf_intermediate_buf); 747 748 if (ring->tx_buffer_info) 749 rte_free(ring->tx_buffer_info); 750 751 if (ring->empty_tx_reqs) 752 rte_free(ring->empty_tx_reqs); 753 754 ring->empty_tx_reqs = NULL; 755 ring->tx_buffer_info = NULL; 756 ring->push_buf_intermediate_buf = NULL; 757 758 ring->configured = 0; 759 760 PMD_DRV_LOG(NOTICE, "TX Queue %d:%d released\n", 761 ring->port_id, ring->id); 762 } 763 764 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 765 { 766 unsigned int i; 767 768 for (i = 0; i < ring->ring_size; ++i) { 769 struct ena_rx_buffer *rx_info = &ring->rx_buffer_info[i]; 770 if (rx_info->mbuf) { 771 rte_mbuf_raw_free(rx_info->mbuf); 772 rx_info->mbuf = NULL; 773 } 774 } 775 } 776 777 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 778 { 779 unsigned int i; 780 781 for (i = 0; i < ring->ring_size; ++i) { 782 struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; 783 784 if (tx_buf->mbuf) { 785 rte_pktmbuf_free(tx_buf->mbuf); 786 tx_buf->mbuf = NULL; 787 } 788 } 789 } 790 791 static int ena_link_update(struct rte_eth_dev *dev, 792 __rte_unused int wait_to_complete) 793 { 794 struct rte_eth_link *link = &dev->data->dev_link; 795 struct ena_adapter *adapter = dev->data->dev_private; 796 797 link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN; 798 link->link_speed = ETH_SPEED_NUM_NONE; 799 link->link_duplex = ETH_LINK_FULL_DUPLEX; 800 801 return 0; 802 } 803 804 static int ena_queue_start_all(struct rte_eth_dev *dev, 805 enum ena_ring_type ring_type) 806 { 807 struct ena_adapter *adapter = dev->data->dev_private; 808 struct ena_ring *queues = NULL; 809 int nb_queues; 810 int i = 0; 811 int rc = 0; 812 813 if (ring_type == ENA_RING_TYPE_RX) { 814 queues = adapter->rx_ring; 815 nb_queues = dev->data->nb_rx_queues; 816 } else { 817 queues = adapter->tx_ring; 818 nb_queues = dev->data->nb_tx_queues; 819 } 820 for (i = 0; i < nb_queues; i++) { 821 if (queues[i].configured) { 822 if (ring_type == ENA_RING_TYPE_RX) { 823 ena_assert_msg( 824 dev->data->rx_queues[i] == &queues[i], 825 "Inconsistent state of rx queues\n"); 826 } else { 827 ena_assert_msg( 828 dev->data->tx_queues[i] == &queues[i], 829 "Inconsistent state of tx queues\n"); 830 } 831 832 rc = ena_queue_start(&queues[i]); 833 834 if (rc) { 835 PMD_INIT_LOG(ERR, 836 "failed to start queue %d type(%d)", 837 i, ring_type); 838 goto err; 839 } 840 } 841 } 842 843 return 0; 844 845 err: 846 while (i--) 847 if (queues[i].configured) 848 ena_queue_stop(&queues[i]); 849 850 return rc; 851 } 852 853 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 854 { 855 uint32_t max_frame_len = adapter->max_mtu; 856 857 if (adapter->edev_data->dev_conf.rxmode.offloads & 858 DEV_RX_OFFLOAD_JUMBO_FRAME) 859 max_frame_len = 860 adapter->edev_data->dev_conf.rxmode.max_rx_pkt_len; 861 862 return max_frame_len; 863 } 864 865 static int ena_check_valid_conf(struct ena_adapter *adapter) 866 { 867 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 868 869 if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) { 870 PMD_INIT_LOG(ERR, "Unsupported MTU of %d. " 871 "max mtu: %d, min mtu: %d", 872 max_frame_len, adapter->max_mtu, ENA_MIN_MTU); 873 return ENA_COM_UNSUPPORTED; 874 } 875 876 return 0; 877 } 878 879 static int 880 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx, 881 bool use_large_llq_hdr) 882 { 883 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; 884 struct ena_com_dev *ena_dev = ctx->ena_dev; 885 uint32_t max_tx_queue_size; 886 uint32_t max_rx_queue_size; 887 888 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 889 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 890 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; 891 max_rx_queue_size = RTE_MIN(max_queue_ext->max_rx_cq_depth, 892 max_queue_ext->max_rx_sq_depth); 893 max_tx_queue_size = max_queue_ext->max_tx_cq_depth; 894 895 if (ena_dev->tx_mem_queue_type == 896 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 897 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 898 llq->max_llq_depth); 899 } else { 900 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 901 max_queue_ext->max_tx_sq_depth); 902 } 903 904 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 905 max_queue_ext->max_per_packet_rx_descs); 906 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 907 max_queue_ext->max_per_packet_tx_descs); 908 } else { 909 struct ena_admin_queue_feature_desc *max_queues = 910 &ctx->get_feat_ctx->max_queues; 911 max_rx_queue_size = RTE_MIN(max_queues->max_cq_depth, 912 max_queues->max_sq_depth); 913 max_tx_queue_size = max_queues->max_cq_depth; 914 915 if (ena_dev->tx_mem_queue_type == 916 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 917 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 918 llq->max_llq_depth); 919 } else { 920 max_tx_queue_size = RTE_MIN(max_tx_queue_size, 921 max_queues->max_sq_depth); 922 } 923 924 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 925 max_queues->max_packet_rx_descs); 926 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 927 max_queues->max_packet_tx_descs); 928 } 929 930 /* Round down to the nearest power of 2 */ 931 max_rx_queue_size = rte_align32prevpow2(max_rx_queue_size); 932 max_tx_queue_size = rte_align32prevpow2(max_tx_queue_size); 933 934 if (use_large_llq_hdr) { 935 if ((llq->entry_size_ctrl_supported & 936 ENA_ADMIN_LIST_ENTRY_SIZE_256B) && 937 (ena_dev->tx_mem_queue_type == 938 ENA_ADMIN_PLACEMENT_POLICY_DEV)) { 939 max_tx_queue_size /= 2; 940 PMD_INIT_LOG(INFO, 941 "Forcing large headers and decreasing maximum TX queue size to %d\n", 942 max_tx_queue_size); 943 } else { 944 PMD_INIT_LOG(ERR, 945 "Forcing large headers failed: LLQ is disabled or device does not support large headers\n"); 946 } 947 } 948 949 if (unlikely(max_rx_queue_size == 0 || max_tx_queue_size == 0)) { 950 PMD_INIT_LOG(ERR, "Invalid queue size"); 951 return -EFAULT; 952 } 953 954 ctx->max_tx_queue_size = max_tx_queue_size; 955 ctx->max_rx_queue_size = max_rx_queue_size; 956 957 return 0; 958 } 959 960 static void ena_stats_restart(struct rte_eth_dev *dev) 961 { 962 struct ena_adapter *adapter = dev->data->dev_private; 963 964 rte_atomic64_init(&adapter->drv_stats->ierrors); 965 rte_atomic64_init(&adapter->drv_stats->oerrors); 966 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 967 adapter->drv_stats->rx_drops = 0; 968 } 969 970 static int ena_stats_get(struct rte_eth_dev *dev, 971 struct rte_eth_stats *stats) 972 { 973 struct ena_admin_basic_stats ena_stats; 974 struct ena_adapter *adapter = dev->data->dev_private; 975 struct ena_com_dev *ena_dev = &adapter->ena_dev; 976 int rc; 977 int i; 978 int max_rings_stats; 979 980 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 981 return -ENOTSUP; 982 983 memset(&ena_stats, 0, sizeof(ena_stats)); 984 985 rte_spinlock_lock(&adapter->admin_lock); 986 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 987 rte_spinlock_unlock(&adapter->admin_lock); 988 if (unlikely(rc)) { 989 PMD_DRV_LOG(ERR, "Could not retrieve statistics from ENA\n"); 990 return rc; 991 } 992 993 /* Set of basic statistics from ENA */ 994 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 995 ena_stats.rx_pkts_low); 996 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 997 ena_stats.tx_pkts_low); 998 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 999 ena_stats.rx_bytes_low); 1000 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 1001 ena_stats.tx_bytes_low); 1002 1003 /* Driver related stats */ 1004 stats->imissed = adapter->drv_stats->rx_drops; 1005 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 1006 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 1007 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 1008 1009 max_rings_stats = RTE_MIN(dev->data->nb_rx_queues, 1010 RTE_ETHDEV_QUEUE_STAT_CNTRS); 1011 for (i = 0; i < max_rings_stats; ++i) { 1012 struct ena_stats_rx *rx_stats = &adapter->rx_ring[i].rx_stats; 1013 1014 stats->q_ibytes[i] = rx_stats->bytes; 1015 stats->q_ipackets[i] = rx_stats->cnt; 1016 stats->q_errors[i] = rx_stats->bad_desc_num + 1017 rx_stats->bad_req_id; 1018 } 1019 1020 max_rings_stats = RTE_MIN(dev->data->nb_tx_queues, 1021 RTE_ETHDEV_QUEUE_STAT_CNTRS); 1022 for (i = 0; i < max_rings_stats; ++i) { 1023 struct ena_stats_tx *tx_stats = &adapter->tx_ring[i].tx_stats; 1024 1025 stats->q_obytes[i] = tx_stats->bytes; 1026 stats->q_opackets[i] = tx_stats->cnt; 1027 } 1028 1029 return 0; 1030 } 1031 1032 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1033 { 1034 struct ena_adapter *adapter; 1035 struct ena_com_dev *ena_dev; 1036 int rc = 0; 1037 1038 ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); 1039 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); 1040 adapter = dev->data->dev_private; 1041 1042 ena_dev = &adapter->ena_dev; 1043 ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); 1044 1045 if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) { 1046 PMD_DRV_LOG(ERR, 1047 "Invalid MTU setting. new_mtu: %d " 1048 "max mtu: %d min mtu: %d\n", 1049 mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU); 1050 return -EINVAL; 1051 } 1052 1053 rc = ena_com_set_dev_mtu(ena_dev, mtu); 1054 if (rc) 1055 PMD_DRV_LOG(ERR, "Could not set MTU: %d\n", mtu); 1056 else 1057 PMD_DRV_LOG(NOTICE, "Set MTU: %d\n", mtu); 1058 1059 return rc; 1060 } 1061 1062 static int ena_start(struct rte_eth_dev *dev) 1063 { 1064 struct ena_adapter *adapter = dev->data->dev_private; 1065 uint64_t ticks; 1066 int rc = 0; 1067 1068 /* Cannot allocate memory in secondary process */ 1069 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1070 PMD_DRV_LOG(WARNING, "dev_start not supported in secondary.\n"); 1071 return -EPERM; 1072 } 1073 1074 rc = ena_check_valid_conf(adapter); 1075 if (rc) 1076 return rc; 1077 1078 rc = ena_queue_start_all(dev, ENA_RING_TYPE_RX); 1079 if (rc) 1080 return rc; 1081 1082 rc = ena_queue_start_all(dev, ENA_RING_TYPE_TX); 1083 if (rc) 1084 goto err_start_tx; 1085 1086 if (adapter->edev_data->dev_conf.rxmode.mq_mode & 1087 ETH_MQ_RX_RSS_FLAG && adapter->edev_data->nb_rx_queues > 0) { 1088 rc = ena_rss_init_default(adapter); 1089 if (rc) 1090 goto err_rss_init; 1091 } 1092 1093 ena_stats_restart(dev); 1094 1095 adapter->timestamp_wd = rte_get_timer_cycles(); 1096 adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; 1097 1098 ticks = rte_get_timer_hz(); 1099 rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(), 1100 ena_timer_wd_callback, dev); 1101 1102 ++adapter->dev_stats.dev_start; 1103 adapter->state = ENA_ADAPTER_STATE_RUNNING; 1104 1105 return 0; 1106 1107 err_rss_init: 1108 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1109 err_start_tx: 1110 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1111 return rc; 1112 } 1113 1114 static int ena_stop(struct rte_eth_dev *dev) 1115 { 1116 struct ena_adapter *adapter = dev->data->dev_private; 1117 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1118 int rc; 1119 1120 /* Cannot free memory in secondary process */ 1121 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1122 PMD_DRV_LOG(WARNING, "dev_stop not supported in secondary.\n"); 1123 return -EPERM; 1124 } 1125 1126 rte_timer_stop_sync(&adapter->timer_wd); 1127 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1128 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1129 1130 if (adapter->trigger_reset) { 1131 rc = ena_com_dev_reset(ena_dev, adapter->reset_reason); 1132 if (rc) 1133 PMD_DRV_LOG(ERR, "Device reset failed rc=%d\n", rc); 1134 } 1135 1136 ++adapter->dev_stats.dev_stop; 1137 adapter->state = ENA_ADAPTER_STATE_STOPPED; 1138 dev->data->dev_started = 0; 1139 1140 return 0; 1141 } 1142 1143 static int ena_create_io_queue(struct ena_ring *ring) 1144 { 1145 struct ena_adapter *adapter; 1146 struct ena_com_dev *ena_dev; 1147 struct ena_com_create_io_ctx ctx = 1148 /* policy set to _HOST just to satisfy icc compiler */ 1149 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1150 0, 0, 0, 0, 0 }; 1151 uint16_t ena_qid; 1152 unsigned int i; 1153 int rc; 1154 1155 adapter = ring->adapter; 1156 ena_dev = &adapter->ena_dev; 1157 1158 if (ring->type == ENA_RING_TYPE_TX) { 1159 ena_qid = ENA_IO_TXQ_IDX(ring->id); 1160 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1161 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1162 for (i = 0; i < ring->ring_size; i++) 1163 ring->empty_tx_reqs[i] = i; 1164 } else { 1165 ena_qid = ENA_IO_RXQ_IDX(ring->id); 1166 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1167 for (i = 0; i < ring->ring_size; i++) 1168 ring->empty_rx_reqs[i] = i; 1169 } 1170 ctx.queue_size = ring->ring_size; 1171 ctx.qid = ena_qid; 1172 ctx.msix_vector = -1; /* interrupts not used */ 1173 ctx.numa_node = ring->numa_socket_id; 1174 1175 rc = ena_com_create_io_queue(ena_dev, &ctx); 1176 if (rc) { 1177 PMD_DRV_LOG(ERR, 1178 "failed to create io queue #%d (qid:%d) rc: %d\n", 1179 ring->id, ena_qid, rc); 1180 return rc; 1181 } 1182 1183 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1184 &ring->ena_com_io_sq, 1185 &ring->ena_com_io_cq); 1186 if (rc) { 1187 PMD_DRV_LOG(ERR, 1188 "Failed to get io queue handlers. queue num %d rc: %d\n", 1189 ring->id, rc); 1190 ena_com_destroy_io_queue(ena_dev, ena_qid); 1191 return rc; 1192 } 1193 1194 if (ring->type == ENA_RING_TYPE_TX) 1195 ena_com_update_numa_node(ring->ena_com_io_cq, ctx.numa_node); 1196 1197 return 0; 1198 } 1199 1200 static void ena_queue_stop(struct ena_ring *ring) 1201 { 1202 struct ena_com_dev *ena_dev = &ring->adapter->ena_dev; 1203 1204 if (ring->type == ENA_RING_TYPE_RX) { 1205 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(ring->id)); 1206 ena_rx_queue_release_bufs(ring); 1207 } else { 1208 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(ring->id)); 1209 ena_tx_queue_release_bufs(ring); 1210 } 1211 } 1212 1213 static void ena_queue_stop_all(struct rte_eth_dev *dev, 1214 enum ena_ring_type ring_type) 1215 { 1216 struct ena_adapter *adapter = dev->data->dev_private; 1217 struct ena_ring *queues = NULL; 1218 uint16_t nb_queues, i; 1219 1220 if (ring_type == ENA_RING_TYPE_RX) { 1221 queues = adapter->rx_ring; 1222 nb_queues = dev->data->nb_rx_queues; 1223 } else { 1224 queues = adapter->tx_ring; 1225 nb_queues = dev->data->nb_tx_queues; 1226 } 1227 1228 for (i = 0; i < nb_queues; ++i) 1229 if (queues[i].configured) 1230 ena_queue_stop(&queues[i]); 1231 } 1232 1233 static int ena_queue_start(struct ena_ring *ring) 1234 { 1235 int rc, bufs_num; 1236 1237 ena_assert_msg(ring->configured == 1, 1238 "Trying to start unconfigured queue\n"); 1239 1240 rc = ena_create_io_queue(ring); 1241 if (rc) { 1242 PMD_INIT_LOG(ERR, "Failed to create IO queue!"); 1243 return rc; 1244 } 1245 1246 ring->next_to_clean = 0; 1247 ring->next_to_use = 0; 1248 1249 if (ring->type == ENA_RING_TYPE_TX) { 1250 ring->tx_stats.available_desc = 1251 ena_com_free_q_entries(ring->ena_com_io_sq); 1252 return 0; 1253 } 1254 1255 bufs_num = ring->ring_size - 1; 1256 rc = ena_populate_rx_queue(ring, bufs_num); 1257 if (rc != bufs_num) { 1258 ena_com_destroy_io_queue(&ring->adapter->ena_dev, 1259 ENA_IO_RXQ_IDX(ring->id)); 1260 PMD_INIT_LOG(ERR, "Failed to populate rx ring !"); 1261 return ENA_COM_FAULT; 1262 } 1263 /* Flush per-core RX buffers pools cache as they can be used on other 1264 * cores as well. 1265 */ 1266 rte_mempool_cache_flush(NULL, ring->mb_pool); 1267 1268 return 0; 1269 } 1270 1271 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 1272 uint16_t queue_idx, 1273 uint16_t nb_desc, 1274 unsigned int socket_id, 1275 const struct rte_eth_txconf *tx_conf) 1276 { 1277 struct ena_ring *txq = NULL; 1278 struct ena_adapter *adapter = dev->data->dev_private; 1279 unsigned int i; 1280 1281 txq = &adapter->tx_ring[queue_idx]; 1282 1283 if (txq->configured) { 1284 PMD_DRV_LOG(CRIT, 1285 "API violation. Queue %d is already configured\n", 1286 queue_idx); 1287 return ENA_COM_FAULT; 1288 } 1289 1290 if (!rte_is_power_of_2(nb_desc)) { 1291 PMD_DRV_LOG(ERR, 1292 "Unsupported size of TX queue: %d is not a power of 2.\n", 1293 nb_desc); 1294 return -EINVAL; 1295 } 1296 1297 if (nb_desc > adapter->max_tx_ring_size) { 1298 PMD_DRV_LOG(ERR, 1299 "Unsupported size of TX queue (max size: %d)\n", 1300 adapter->max_tx_ring_size); 1301 return -EINVAL; 1302 } 1303 1304 txq->port_id = dev->data->port_id; 1305 txq->next_to_clean = 0; 1306 txq->next_to_use = 0; 1307 txq->ring_size = nb_desc; 1308 txq->size_mask = nb_desc - 1; 1309 txq->numa_socket_id = socket_id; 1310 txq->pkts_without_db = false; 1311 1312 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1313 sizeof(struct ena_tx_buffer) * 1314 txq->ring_size, 1315 RTE_CACHE_LINE_SIZE); 1316 if (!txq->tx_buffer_info) { 1317 PMD_DRV_LOG(ERR, "failed to alloc mem for tx buffer info\n"); 1318 return -ENOMEM; 1319 } 1320 1321 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1322 sizeof(u16) * txq->ring_size, 1323 RTE_CACHE_LINE_SIZE); 1324 if (!txq->empty_tx_reqs) { 1325 PMD_DRV_LOG(ERR, "failed to alloc mem for tx reqs\n"); 1326 rte_free(txq->tx_buffer_info); 1327 return -ENOMEM; 1328 } 1329 1330 txq->push_buf_intermediate_buf = 1331 rte_zmalloc("txq->push_buf_intermediate_buf", 1332 txq->tx_max_header_size, 1333 RTE_CACHE_LINE_SIZE); 1334 if (!txq->push_buf_intermediate_buf) { 1335 PMD_DRV_LOG(ERR, "failed to alloc push buff for LLQ\n"); 1336 rte_free(txq->tx_buffer_info); 1337 rte_free(txq->empty_tx_reqs); 1338 return -ENOMEM; 1339 } 1340 1341 for (i = 0; i < txq->ring_size; i++) 1342 txq->empty_tx_reqs[i] = i; 1343 1344 if (tx_conf != NULL) { 1345 txq->offloads = 1346 tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1347 } 1348 /* Store pointer to this queue in upper layer */ 1349 txq->configured = 1; 1350 dev->data->tx_queues[queue_idx] = txq; 1351 1352 return 0; 1353 } 1354 1355 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1356 uint16_t queue_idx, 1357 uint16_t nb_desc, 1358 unsigned int socket_id, 1359 __rte_unused const struct rte_eth_rxconf *rx_conf, 1360 struct rte_mempool *mp) 1361 { 1362 struct ena_adapter *adapter = dev->data->dev_private; 1363 struct ena_ring *rxq = NULL; 1364 size_t buffer_size; 1365 int i; 1366 1367 rxq = &adapter->rx_ring[queue_idx]; 1368 if (rxq->configured) { 1369 PMD_DRV_LOG(CRIT, 1370 "API violation. Queue %d is already configured\n", 1371 queue_idx); 1372 return ENA_COM_FAULT; 1373 } 1374 1375 if (!rte_is_power_of_2(nb_desc)) { 1376 PMD_DRV_LOG(ERR, 1377 "Unsupported size of RX queue: %d is not a power of 2.\n", 1378 nb_desc); 1379 return -EINVAL; 1380 } 1381 1382 if (nb_desc > adapter->max_rx_ring_size) { 1383 PMD_DRV_LOG(ERR, 1384 "Unsupported size of RX queue (max size: %d)\n", 1385 adapter->max_rx_ring_size); 1386 return -EINVAL; 1387 } 1388 1389 /* ENA isn't supporting buffers smaller than 1400 bytes */ 1390 buffer_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM; 1391 if (buffer_size < ENA_RX_BUF_MIN_SIZE) { 1392 PMD_DRV_LOG(ERR, 1393 "Unsupported size of RX buffer: %zu (min size: %d)\n", 1394 buffer_size, ENA_RX_BUF_MIN_SIZE); 1395 return -EINVAL; 1396 } 1397 1398 rxq->port_id = dev->data->port_id; 1399 rxq->next_to_clean = 0; 1400 rxq->next_to_use = 0; 1401 rxq->ring_size = nb_desc; 1402 rxq->size_mask = nb_desc - 1; 1403 rxq->numa_socket_id = socket_id; 1404 rxq->mb_pool = mp; 1405 1406 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1407 sizeof(struct ena_rx_buffer) * nb_desc, 1408 RTE_CACHE_LINE_SIZE); 1409 if (!rxq->rx_buffer_info) { 1410 PMD_DRV_LOG(ERR, "failed to alloc mem for rx buffer info\n"); 1411 return -ENOMEM; 1412 } 1413 1414 rxq->rx_refill_buffer = rte_zmalloc("rxq->rx_refill_buffer", 1415 sizeof(struct rte_mbuf *) * nb_desc, 1416 RTE_CACHE_LINE_SIZE); 1417 1418 if (!rxq->rx_refill_buffer) { 1419 PMD_DRV_LOG(ERR, "failed to alloc mem for rx refill buffer\n"); 1420 rte_free(rxq->rx_buffer_info); 1421 rxq->rx_buffer_info = NULL; 1422 return -ENOMEM; 1423 } 1424 1425 rxq->empty_rx_reqs = rte_zmalloc("rxq->empty_rx_reqs", 1426 sizeof(uint16_t) * nb_desc, 1427 RTE_CACHE_LINE_SIZE); 1428 if (!rxq->empty_rx_reqs) { 1429 PMD_DRV_LOG(ERR, "failed to alloc mem for empty rx reqs\n"); 1430 rte_free(rxq->rx_buffer_info); 1431 rxq->rx_buffer_info = NULL; 1432 rte_free(rxq->rx_refill_buffer); 1433 rxq->rx_refill_buffer = NULL; 1434 return -ENOMEM; 1435 } 1436 1437 for (i = 0; i < nb_desc; i++) 1438 rxq->empty_rx_reqs[i] = i; 1439 1440 /* Store pointer to this queue in upper layer */ 1441 rxq->configured = 1; 1442 dev->data->rx_queues[queue_idx] = rxq; 1443 1444 return 0; 1445 } 1446 1447 static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, 1448 struct rte_mbuf *mbuf, uint16_t id) 1449 { 1450 struct ena_com_buf ebuf; 1451 int rc; 1452 1453 /* prepare physical address for DMA transaction */ 1454 ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; 1455 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1456 1457 /* pass resource to device */ 1458 rc = ena_com_add_single_rx_desc(io_sq, &ebuf, id); 1459 if (unlikely(rc != 0)) 1460 PMD_DRV_LOG(WARNING, "failed adding rx desc\n"); 1461 1462 return rc; 1463 } 1464 1465 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1466 { 1467 unsigned int i; 1468 int rc; 1469 uint16_t next_to_use = rxq->next_to_use; 1470 uint16_t in_use, req_id; 1471 struct rte_mbuf **mbufs = rxq->rx_refill_buffer; 1472 1473 if (unlikely(!count)) 1474 return 0; 1475 1476 in_use = rxq->ring_size - 1 - 1477 ena_com_free_q_entries(rxq->ena_com_io_sq); 1478 ena_assert_msg(((in_use + count) < rxq->ring_size), 1479 "bad ring state\n"); 1480 1481 /* get resources for incoming packets */ 1482 rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, count); 1483 if (unlikely(rc < 0)) { 1484 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1485 ++rxq->rx_stats.mbuf_alloc_fail; 1486 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1487 return 0; 1488 } 1489 1490 for (i = 0; i < count; i++) { 1491 struct rte_mbuf *mbuf = mbufs[i]; 1492 struct ena_rx_buffer *rx_info; 1493 1494 if (likely((i + 4) < count)) 1495 rte_prefetch0(mbufs[i + 4]); 1496 1497 req_id = rxq->empty_rx_reqs[next_to_use]; 1498 rx_info = &rxq->rx_buffer_info[req_id]; 1499 1500 rc = ena_add_single_rx_desc(rxq->ena_com_io_sq, mbuf, req_id); 1501 if (unlikely(rc != 0)) 1502 break; 1503 1504 rx_info->mbuf = mbuf; 1505 next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, rxq->size_mask); 1506 } 1507 1508 if (unlikely(i < count)) { 1509 PMD_DRV_LOG(WARNING, "refilled rx qid %d with only %d " 1510 "buffers (from %d)\n", rxq->id, i, count); 1511 rte_pktmbuf_free_bulk(&mbufs[i], count - i); 1512 ++rxq->rx_stats.refill_partial; 1513 } 1514 1515 /* When we submitted free recources to device... */ 1516 if (likely(i > 0)) { 1517 /* ...let HW know that it can fill buffers with data. */ 1518 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1519 1520 rxq->next_to_use = next_to_use; 1521 } 1522 1523 return i; 1524 } 1525 1526 static int ena_device_init(struct ena_com_dev *ena_dev, 1527 struct rte_pci_device *pdev, 1528 struct ena_com_dev_get_features_ctx *get_feat_ctx, 1529 bool *wd_state) 1530 { 1531 uint32_t aenq_groups; 1532 int rc; 1533 bool readless_supported; 1534 1535 /* Initialize mmio registers */ 1536 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1537 if (rc) { 1538 PMD_DRV_LOG(ERR, "failed to init mmio read less\n"); 1539 return rc; 1540 } 1541 1542 /* The PCIe configuration space revision id indicate if mmio reg 1543 * read is disabled. 1544 */ 1545 readless_supported = !(pdev->id.class_id & ENA_MMIO_DISABLE_REG_READ); 1546 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1547 1548 /* reset device */ 1549 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 1550 if (rc) { 1551 PMD_DRV_LOG(ERR, "cannot reset device\n"); 1552 goto err_mmio_read_less; 1553 } 1554 1555 /* check FW version */ 1556 rc = ena_com_validate_version(ena_dev); 1557 if (rc) { 1558 PMD_DRV_LOG(ERR, "device version is too low\n"); 1559 goto err_mmio_read_less; 1560 } 1561 1562 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1563 1564 /* ENA device administration layer init */ 1565 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 1566 if (rc) { 1567 PMD_DRV_LOG(ERR, 1568 "cannot initialize ena admin queue with device\n"); 1569 goto err_mmio_read_less; 1570 } 1571 1572 /* To enable the msix interrupts the driver needs to know the number 1573 * of queues. So the driver uses polling mode to retrieve this 1574 * information. 1575 */ 1576 ena_com_set_admin_polling_mode(ena_dev, true); 1577 1578 ena_config_host_info(ena_dev); 1579 1580 /* Get Device Attributes and features */ 1581 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1582 if (rc) { 1583 PMD_DRV_LOG(ERR, 1584 "cannot get attribute for ena device rc= %d\n", rc); 1585 goto err_admin_init; 1586 } 1587 1588 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 1589 BIT(ENA_ADMIN_NOTIFICATION) | 1590 BIT(ENA_ADMIN_KEEP_ALIVE) | 1591 BIT(ENA_ADMIN_FATAL_ERROR) | 1592 BIT(ENA_ADMIN_WARNING); 1593 1594 aenq_groups &= get_feat_ctx->aenq.supported_groups; 1595 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 1596 if (rc) { 1597 PMD_DRV_LOG(ERR, "Cannot configure aenq groups rc: %d\n", rc); 1598 goto err_admin_init; 1599 } 1600 1601 *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 1602 1603 return 0; 1604 1605 err_admin_init: 1606 ena_com_admin_destroy(ena_dev); 1607 1608 err_mmio_read_less: 1609 ena_com_mmio_reg_read_request_destroy(ena_dev); 1610 1611 return rc; 1612 } 1613 1614 static void ena_interrupt_handler_rte(void *cb_arg) 1615 { 1616 struct rte_eth_dev *dev = cb_arg; 1617 struct ena_adapter *adapter = dev->data->dev_private; 1618 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1619 1620 ena_com_admin_q_comp_intr_handler(ena_dev); 1621 if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED)) 1622 ena_com_aenq_intr_handler(ena_dev, dev); 1623 } 1624 1625 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 1626 { 1627 if (!adapter->wd_state) 1628 return; 1629 1630 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 1631 return; 1632 1633 if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >= 1634 adapter->keep_alive_timeout)) { 1635 PMD_DRV_LOG(ERR, "Keep alive timeout\n"); 1636 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; 1637 adapter->trigger_reset = true; 1638 ++adapter->dev_stats.wd_expired; 1639 } 1640 } 1641 1642 /* Check if admin queue is enabled */ 1643 static void check_for_admin_com_state(struct ena_adapter *adapter) 1644 { 1645 if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) { 1646 PMD_DRV_LOG(ERR, "ENA admin queue is not in running state!\n"); 1647 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; 1648 adapter->trigger_reset = true; 1649 } 1650 } 1651 1652 static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer, 1653 void *arg) 1654 { 1655 struct rte_eth_dev *dev = arg; 1656 struct ena_adapter *adapter = dev->data->dev_private; 1657 1658 check_for_missing_keep_alive(adapter); 1659 check_for_admin_com_state(adapter); 1660 1661 if (unlikely(adapter->trigger_reset)) { 1662 PMD_DRV_LOG(ERR, "Trigger reset is on\n"); 1663 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 1664 NULL); 1665 } 1666 } 1667 1668 static inline void 1669 set_default_llq_configurations(struct ena_llq_configurations *llq_config, 1670 struct ena_admin_feature_llq_desc *llq, 1671 bool use_large_llq_hdr) 1672 { 1673 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 1674 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 1675 llq_config->llq_num_decs_before_header = 1676 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 1677 1678 if (use_large_llq_hdr && 1679 (llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B)) { 1680 llq_config->llq_ring_entry_size = 1681 ENA_ADMIN_LIST_ENTRY_SIZE_256B; 1682 llq_config->llq_ring_entry_size_value = 256; 1683 } else { 1684 llq_config->llq_ring_entry_size = 1685 ENA_ADMIN_LIST_ENTRY_SIZE_128B; 1686 llq_config->llq_ring_entry_size_value = 128; 1687 } 1688 } 1689 1690 static int 1691 ena_set_queues_placement_policy(struct ena_adapter *adapter, 1692 struct ena_com_dev *ena_dev, 1693 struct ena_admin_feature_llq_desc *llq, 1694 struct ena_llq_configurations *llq_default_configurations) 1695 { 1696 int rc; 1697 u32 llq_feature_mask; 1698 1699 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 1700 if (!(ena_dev->supported_features & llq_feature_mask)) { 1701 PMD_DRV_LOG(INFO, 1702 "LLQ is not supported. Fallback to host mode policy.\n"); 1703 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1704 return 0; 1705 } 1706 1707 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 1708 if (unlikely(rc)) { 1709 PMD_INIT_LOG(WARNING, "Failed to config dev mode. " 1710 "Fallback to host mode policy."); 1711 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1712 return 0; 1713 } 1714 1715 /* Nothing to config, exit */ 1716 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 1717 return 0; 1718 1719 if (!adapter->dev_mem_base) { 1720 PMD_DRV_LOG(ERR, "Unable to access LLQ bar resource. " 1721 "Fallback to host mode policy.\n."); 1722 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1723 return 0; 1724 } 1725 1726 ena_dev->mem_bar = adapter->dev_mem_base; 1727 1728 return 0; 1729 } 1730 1731 static uint32_t ena_calc_max_io_queue_num(struct ena_com_dev *ena_dev, 1732 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1733 { 1734 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues; 1735 1736 /* Regular queues capabilities */ 1737 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 1738 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 1739 &get_feat_ctx->max_queue_ext.max_queue_ext; 1740 io_rx_num = RTE_MIN(max_queue_ext->max_rx_sq_num, 1741 max_queue_ext->max_rx_cq_num); 1742 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 1743 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 1744 } else { 1745 struct ena_admin_queue_feature_desc *max_queues = 1746 &get_feat_ctx->max_queues; 1747 io_tx_sq_num = max_queues->max_sq_num; 1748 io_tx_cq_num = max_queues->max_cq_num; 1749 io_rx_num = RTE_MIN(io_tx_sq_num, io_tx_cq_num); 1750 } 1751 1752 /* In case of LLQ use the llq number in the get feature cmd */ 1753 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 1754 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 1755 1756 max_num_io_queues = RTE_MIN(ENA_MAX_NUM_IO_QUEUES, io_rx_num); 1757 max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_sq_num); 1758 max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_cq_num); 1759 1760 if (unlikely(max_num_io_queues == 0)) { 1761 PMD_DRV_LOG(ERR, "Number of IO queues should not be 0\n"); 1762 return -EFAULT; 1763 } 1764 1765 return max_num_io_queues; 1766 } 1767 1768 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1769 { 1770 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 1771 struct rte_pci_device *pci_dev; 1772 struct rte_intr_handle *intr_handle; 1773 struct ena_adapter *adapter = eth_dev->data->dev_private; 1774 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1775 struct ena_com_dev_get_features_ctx get_feat_ctx; 1776 struct ena_llq_configurations llq_config; 1777 const char *queue_type_str; 1778 uint32_t max_num_io_queues; 1779 int rc; 1780 static int adapters_found; 1781 bool disable_meta_caching; 1782 bool wd_state = false; 1783 1784 eth_dev->dev_ops = &ena_dev_ops; 1785 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1786 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1787 eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; 1788 1789 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1790 return 0; 1791 1792 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 1793 1794 memset(adapter, 0, sizeof(struct ena_adapter)); 1795 ena_dev = &adapter->ena_dev; 1796 1797 adapter->edev_data = eth_dev->data; 1798 1799 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1800 1801 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", 1802 pci_dev->addr.domain, 1803 pci_dev->addr.bus, 1804 pci_dev->addr.devid, 1805 pci_dev->addr.function); 1806 1807 intr_handle = &pci_dev->intr_handle; 1808 1809 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1810 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1811 1812 if (!adapter->regs) { 1813 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)", 1814 ENA_REGS_BAR); 1815 return -ENXIO; 1816 } 1817 1818 ena_dev->reg_bar = adapter->regs; 1819 /* This is a dummy pointer for ena_com functions. */ 1820 ena_dev->dmadev = adapter; 1821 1822 adapter->id_number = adapters_found; 1823 1824 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1825 adapter->id_number); 1826 1827 rc = ena_parse_devargs(adapter, pci_dev->device.devargs); 1828 if (rc != 0) { 1829 PMD_INIT_LOG(CRIT, "Failed to parse devargs\n"); 1830 goto err; 1831 } 1832 1833 /* device specific initialization routine */ 1834 rc = ena_device_init(ena_dev, pci_dev, &get_feat_ctx, &wd_state); 1835 if (rc) { 1836 PMD_INIT_LOG(CRIT, "Failed to init ENA device"); 1837 goto err; 1838 } 1839 adapter->wd_state = wd_state; 1840 1841 set_default_llq_configurations(&llq_config, &get_feat_ctx.llq, 1842 adapter->use_large_llq_hdr); 1843 rc = ena_set_queues_placement_policy(adapter, ena_dev, 1844 &get_feat_ctx.llq, &llq_config); 1845 if (unlikely(rc)) { 1846 PMD_INIT_LOG(CRIT, "Failed to set placement policy"); 1847 return rc; 1848 } 1849 1850 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 1851 queue_type_str = "Regular"; 1852 else 1853 queue_type_str = "Low latency"; 1854 PMD_DRV_LOG(INFO, "Placement policy: %s\n", queue_type_str); 1855 1856 calc_queue_ctx.ena_dev = ena_dev; 1857 calc_queue_ctx.get_feat_ctx = &get_feat_ctx; 1858 1859 max_num_io_queues = ena_calc_max_io_queue_num(ena_dev, &get_feat_ctx); 1860 rc = ena_calc_io_queue_size(&calc_queue_ctx, 1861 adapter->use_large_llq_hdr); 1862 if (unlikely((rc != 0) || (max_num_io_queues == 0))) { 1863 rc = -EFAULT; 1864 goto err_device_destroy; 1865 } 1866 1867 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; 1868 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; 1869 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 1870 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 1871 adapter->max_num_io_queues = max_num_io_queues; 1872 1873 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1874 disable_meta_caching = 1875 !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags & 1876 BIT(ENA_ADMIN_DISABLE_META_CACHING)); 1877 } else { 1878 disable_meta_caching = false; 1879 } 1880 1881 /* prepare ring structures */ 1882 ena_init_rings(adapter, disable_meta_caching); 1883 1884 ena_config_debug_area(adapter); 1885 1886 /* Set max MTU for this device */ 1887 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1888 1889 /* set device support for offloads */ 1890 adapter->offloads.tso4_supported = (get_feat_ctx.offload.tx & 1891 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0; 1892 adapter->offloads.tx_csum_supported = (get_feat_ctx.offload.tx & 1893 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) != 0; 1894 adapter->offloads.rx_csum_supported = 1895 (get_feat_ctx.offload.rx_supported & 1896 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) != 0; 1897 1898 /* Copy MAC address and point DPDK to it */ 1899 eth_dev->data->mac_addrs = (struct rte_ether_addr *)adapter->mac_addr; 1900 rte_ether_addr_copy((struct rte_ether_addr *) 1901 get_feat_ctx.dev_attr.mac_addr, 1902 (struct rte_ether_addr *)adapter->mac_addr); 1903 1904 adapter->drv_stats = rte_zmalloc("adapter stats", 1905 sizeof(*adapter->drv_stats), 1906 RTE_CACHE_LINE_SIZE); 1907 if (!adapter->drv_stats) { 1908 PMD_DRV_LOG(ERR, "failed to alloc mem for adapter stats\n"); 1909 rc = -ENOMEM; 1910 goto err_delete_debug_area; 1911 } 1912 1913 rte_spinlock_init(&adapter->admin_lock); 1914 1915 rte_intr_callback_register(intr_handle, 1916 ena_interrupt_handler_rte, 1917 eth_dev); 1918 rte_intr_enable(intr_handle); 1919 ena_com_set_admin_polling_mode(ena_dev, false); 1920 ena_com_admin_aenq_enable(ena_dev); 1921 1922 if (adapters_found == 0) 1923 rte_timer_subsystem_init(); 1924 rte_timer_init(&adapter->timer_wd); 1925 1926 adapters_found++; 1927 adapter->state = ENA_ADAPTER_STATE_INIT; 1928 1929 return 0; 1930 1931 err_delete_debug_area: 1932 ena_com_delete_debug_area(ena_dev); 1933 1934 err_device_destroy: 1935 ena_com_delete_host_info(ena_dev); 1936 ena_com_admin_destroy(ena_dev); 1937 1938 err: 1939 return rc; 1940 } 1941 1942 static void ena_destroy_device(struct rte_eth_dev *eth_dev) 1943 { 1944 struct ena_adapter *adapter = eth_dev->data->dev_private; 1945 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1946 1947 if (adapter->state == ENA_ADAPTER_STATE_FREE) 1948 return; 1949 1950 ena_com_set_admin_running_state(ena_dev, false); 1951 1952 if (adapter->state != ENA_ADAPTER_STATE_CLOSED) 1953 ena_close(eth_dev); 1954 1955 ena_com_delete_debug_area(ena_dev); 1956 ena_com_delete_host_info(ena_dev); 1957 1958 ena_com_abort_admin_commands(ena_dev); 1959 ena_com_wait_for_abort_completion(ena_dev); 1960 ena_com_admin_destroy(ena_dev); 1961 ena_com_mmio_reg_read_request_destroy(ena_dev); 1962 1963 adapter->state = ENA_ADAPTER_STATE_FREE; 1964 } 1965 1966 static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev) 1967 { 1968 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1969 return 0; 1970 1971 ena_destroy_device(eth_dev); 1972 1973 return 0; 1974 } 1975 1976 static int ena_dev_configure(struct rte_eth_dev *dev) 1977 { 1978 struct ena_adapter *adapter = dev->data->dev_private; 1979 1980 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1981 1982 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 1983 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1984 1985 adapter->tx_selected_offloads = dev->data->dev_conf.txmode.offloads; 1986 adapter->rx_selected_offloads = dev->data->dev_conf.rxmode.offloads; 1987 return 0; 1988 } 1989 1990 static void ena_init_rings(struct ena_adapter *adapter, 1991 bool disable_meta_caching) 1992 { 1993 size_t i; 1994 1995 for (i = 0; i < adapter->max_num_io_queues; i++) { 1996 struct ena_ring *ring = &adapter->tx_ring[i]; 1997 1998 ring->configured = 0; 1999 ring->type = ENA_RING_TYPE_TX; 2000 ring->adapter = adapter; 2001 ring->id = i; 2002 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 2003 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 2004 ring->sgl_size = adapter->max_tx_sgl_size; 2005 ring->disable_meta_caching = disable_meta_caching; 2006 } 2007 2008 for (i = 0; i < adapter->max_num_io_queues; i++) { 2009 struct ena_ring *ring = &adapter->rx_ring[i]; 2010 2011 ring->configured = 0; 2012 ring->type = ENA_RING_TYPE_RX; 2013 ring->adapter = adapter; 2014 ring->id = i; 2015 ring->sgl_size = adapter->max_rx_sgl_size; 2016 } 2017 } 2018 2019 static int ena_infos_get(struct rte_eth_dev *dev, 2020 struct rte_eth_dev_info *dev_info) 2021 { 2022 struct ena_adapter *adapter; 2023 struct ena_com_dev *ena_dev; 2024 uint64_t rx_feat = 0, tx_feat = 0; 2025 2026 ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); 2027 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); 2028 adapter = dev->data->dev_private; 2029 2030 ena_dev = &adapter->ena_dev; 2031 ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); 2032 2033 dev_info->speed_capa = 2034 ETH_LINK_SPEED_1G | 2035 ETH_LINK_SPEED_2_5G | 2036 ETH_LINK_SPEED_5G | 2037 ETH_LINK_SPEED_10G | 2038 ETH_LINK_SPEED_25G | 2039 ETH_LINK_SPEED_40G | 2040 ETH_LINK_SPEED_50G | 2041 ETH_LINK_SPEED_100G; 2042 2043 /* Set Tx & Rx features available for device */ 2044 if (adapter->offloads.tso4_supported) 2045 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 2046 2047 if (adapter->offloads.tx_csum_supported) 2048 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 2049 DEV_TX_OFFLOAD_UDP_CKSUM | 2050 DEV_TX_OFFLOAD_TCP_CKSUM; 2051 2052 if (adapter->offloads.rx_csum_supported) 2053 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 2054 DEV_RX_OFFLOAD_UDP_CKSUM | 2055 DEV_RX_OFFLOAD_TCP_CKSUM; 2056 2057 rx_feat |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2058 2059 /* Inform framework about available features */ 2060 dev_info->rx_offload_capa = rx_feat; 2061 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_RSS_HASH; 2062 dev_info->rx_queue_offload_capa = rx_feat; 2063 dev_info->tx_offload_capa = tx_feat; 2064 dev_info->tx_queue_offload_capa = tx_feat; 2065 2066 dev_info->flow_type_rss_offloads = ETH_RSS_IP | ETH_RSS_TCP | 2067 ETH_RSS_UDP; 2068 2069 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 2070 dev_info->max_rx_pktlen = adapter->max_mtu; 2071 dev_info->max_mac_addrs = 1; 2072 2073 dev_info->max_rx_queues = adapter->max_num_io_queues; 2074 dev_info->max_tx_queues = adapter->max_num_io_queues; 2075 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 2076 2077 adapter->tx_supported_offloads = tx_feat; 2078 adapter->rx_supported_offloads = rx_feat; 2079 2080 dev_info->rx_desc_lim.nb_max = adapter->max_rx_ring_size; 2081 dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC; 2082 dev_info->rx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2083 adapter->max_rx_sgl_size); 2084 dev_info->rx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2085 adapter->max_rx_sgl_size); 2086 2087 dev_info->tx_desc_lim.nb_max = adapter->max_tx_ring_size; 2088 dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC; 2089 dev_info->tx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2090 adapter->max_tx_sgl_size); 2091 dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 2092 adapter->max_tx_sgl_size); 2093 2094 dev_info->default_rxportconf.ring_size = ENA_DEFAULT_RING_SIZE; 2095 dev_info->default_txportconf.ring_size = ENA_DEFAULT_RING_SIZE; 2096 2097 return 0; 2098 } 2099 2100 static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len) 2101 { 2102 mbuf->data_len = len; 2103 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2104 mbuf->refcnt = 1; 2105 mbuf->next = NULL; 2106 } 2107 2108 static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, 2109 struct ena_com_rx_buf_info *ena_bufs, 2110 uint32_t descs, 2111 uint16_t *next_to_clean, 2112 uint8_t offset) 2113 { 2114 struct rte_mbuf *mbuf; 2115 struct rte_mbuf *mbuf_head; 2116 struct ena_rx_buffer *rx_info; 2117 int rc; 2118 uint16_t ntc, len, req_id, buf = 0; 2119 2120 if (unlikely(descs == 0)) 2121 return NULL; 2122 2123 ntc = *next_to_clean; 2124 2125 len = ena_bufs[buf].len; 2126 req_id = ena_bufs[buf].req_id; 2127 2128 rx_info = &rx_ring->rx_buffer_info[req_id]; 2129 2130 mbuf = rx_info->mbuf; 2131 RTE_ASSERT(mbuf != NULL); 2132 2133 ena_init_rx_mbuf(mbuf, len); 2134 2135 /* Fill the mbuf head with the data specific for 1st segment. */ 2136 mbuf_head = mbuf; 2137 mbuf_head->nb_segs = descs; 2138 mbuf_head->port = rx_ring->port_id; 2139 mbuf_head->pkt_len = len; 2140 mbuf_head->data_off += offset; 2141 2142 rx_info->mbuf = NULL; 2143 rx_ring->empty_rx_reqs[ntc] = req_id; 2144 ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); 2145 2146 while (--descs) { 2147 ++buf; 2148 len = ena_bufs[buf].len; 2149 req_id = ena_bufs[buf].req_id; 2150 2151 rx_info = &rx_ring->rx_buffer_info[req_id]; 2152 RTE_ASSERT(rx_info->mbuf != NULL); 2153 2154 if (unlikely(len == 0)) { 2155 /* 2156 * Some devices can pass descriptor with the length 0. 2157 * To avoid confusion, the PMD is simply putting the 2158 * descriptor back, as it was never used. We'll avoid 2159 * mbuf allocation that way. 2160 */ 2161 rc = ena_add_single_rx_desc(rx_ring->ena_com_io_sq, 2162 rx_info->mbuf, req_id); 2163 if (unlikely(rc != 0)) { 2164 /* Free the mbuf in case of an error. */ 2165 rte_mbuf_raw_free(rx_info->mbuf); 2166 } else { 2167 /* 2168 * If there was no error, just exit the loop as 2169 * 0 length descriptor is always the last one. 2170 */ 2171 break; 2172 } 2173 } else { 2174 /* Create an mbuf chain. */ 2175 mbuf->next = rx_info->mbuf; 2176 mbuf = mbuf->next; 2177 2178 ena_init_rx_mbuf(mbuf, len); 2179 mbuf_head->pkt_len += len; 2180 } 2181 2182 /* 2183 * Mark the descriptor as depleted and perform necessary 2184 * cleanup. 2185 * This code will execute in two cases: 2186 * 1. Descriptor len was greater than 0 - normal situation. 2187 * 2. Descriptor len was 0 and we failed to add the descriptor 2188 * to the device. In that situation, we should try to add 2189 * the mbuf again in the populate routine and mark the 2190 * descriptor as used up by the device. 2191 */ 2192 rx_info->mbuf = NULL; 2193 rx_ring->empty_rx_reqs[ntc] = req_id; 2194 ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); 2195 } 2196 2197 *next_to_clean = ntc; 2198 2199 return mbuf_head; 2200 } 2201 2202 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 2203 uint16_t nb_pkts) 2204 { 2205 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 2206 unsigned int free_queue_entries; 2207 unsigned int refill_threshold; 2208 uint16_t next_to_clean = rx_ring->next_to_clean; 2209 uint16_t descs_in_use; 2210 struct rte_mbuf *mbuf; 2211 uint16_t completed; 2212 struct ena_com_rx_ctx ena_rx_ctx; 2213 int i, rc = 0; 2214 2215 /* Check adapter state */ 2216 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2217 PMD_DRV_LOG(ALERT, 2218 "Trying to receive pkts while device is NOT running\n"); 2219 return 0; 2220 } 2221 2222 descs_in_use = rx_ring->ring_size - 2223 ena_com_free_q_entries(rx_ring->ena_com_io_sq) - 1; 2224 nb_pkts = RTE_MIN(descs_in_use, nb_pkts); 2225 2226 for (completed = 0; completed < nb_pkts; completed++) { 2227 ena_rx_ctx.max_bufs = rx_ring->sgl_size; 2228 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 2229 ena_rx_ctx.descs = 0; 2230 ena_rx_ctx.pkt_offset = 0; 2231 /* receive packet context */ 2232 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 2233 rx_ring->ena_com_io_sq, 2234 &ena_rx_ctx); 2235 if (unlikely(rc)) { 2236 PMD_DRV_LOG(ERR, "ena_com_rx_pkt error %d\n", rc); 2237 if (rc == ENA_COM_NO_SPACE) { 2238 ++rx_ring->rx_stats.bad_desc_num; 2239 rx_ring->adapter->reset_reason = 2240 ENA_REGS_RESET_TOO_MANY_RX_DESCS; 2241 } else { 2242 ++rx_ring->rx_stats.bad_req_id; 2243 rx_ring->adapter->reset_reason = 2244 ENA_REGS_RESET_INV_RX_REQ_ID; 2245 } 2246 rx_ring->adapter->trigger_reset = true; 2247 return 0; 2248 } 2249 2250 mbuf = ena_rx_mbuf(rx_ring, 2251 ena_rx_ctx.ena_bufs, 2252 ena_rx_ctx.descs, 2253 &next_to_clean, 2254 ena_rx_ctx.pkt_offset); 2255 if (unlikely(mbuf == NULL)) { 2256 for (i = 0; i < ena_rx_ctx.descs; ++i) { 2257 rx_ring->empty_rx_reqs[next_to_clean] = 2258 rx_ring->ena_bufs[i].req_id; 2259 next_to_clean = ENA_IDX_NEXT_MASKED( 2260 next_to_clean, rx_ring->size_mask); 2261 } 2262 break; 2263 } 2264 2265 /* fill mbuf attributes if any */ 2266 ena_rx_mbuf_prepare(mbuf, &ena_rx_ctx); 2267 2268 if (unlikely(mbuf->ol_flags & 2269 (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD))) { 2270 rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors); 2271 ++rx_ring->rx_stats.bad_csum; 2272 } 2273 2274 rx_pkts[completed] = mbuf; 2275 rx_ring->rx_stats.bytes += mbuf->pkt_len; 2276 } 2277 2278 rx_ring->rx_stats.cnt += completed; 2279 rx_ring->next_to_clean = next_to_clean; 2280 2281 free_queue_entries = ena_com_free_q_entries(rx_ring->ena_com_io_sq); 2282 refill_threshold = 2283 RTE_MIN(rx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, 2284 (unsigned int)ENA_REFILL_THRESH_PACKET); 2285 2286 /* Burst refill to save doorbells, memory barriers, const interval */ 2287 if (free_queue_entries > refill_threshold) { 2288 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); 2289 ena_populate_rx_queue(rx_ring, free_queue_entries); 2290 } 2291 2292 return completed; 2293 } 2294 2295 static uint16_t 2296 eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2297 uint16_t nb_pkts) 2298 { 2299 int32_t ret; 2300 uint32_t i; 2301 struct rte_mbuf *m; 2302 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2303 struct rte_ipv4_hdr *ip_hdr; 2304 uint64_t ol_flags; 2305 uint16_t frag_field; 2306 2307 for (i = 0; i != nb_pkts; i++) { 2308 m = tx_pkts[i]; 2309 ol_flags = m->ol_flags; 2310 2311 if (!(ol_flags & PKT_TX_IPV4)) 2312 continue; 2313 2314 /* If there was not L2 header length specified, assume it is 2315 * length of the ethernet header. 2316 */ 2317 if (unlikely(m->l2_len == 0)) 2318 m->l2_len = sizeof(struct rte_ether_hdr); 2319 2320 ip_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, 2321 m->l2_len); 2322 frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); 2323 2324 if ((frag_field & RTE_IPV4_HDR_DF_FLAG) != 0) { 2325 m->packet_type |= RTE_PTYPE_L4_NONFRAG; 2326 2327 /* If IPv4 header has DF flag enabled and TSO support is 2328 * disabled, partial chcecksum should not be calculated. 2329 */ 2330 if (!tx_ring->adapter->offloads.tso4_supported) 2331 continue; 2332 } 2333 2334 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || 2335 (ol_flags & PKT_TX_L4_MASK) == 2336 PKT_TX_SCTP_CKSUM) { 2337 rte_errno = ENOTSUP; 2338 return i; 2339 } 2340 2341 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 2342 ret = rte_validate_tx_offload(m); 2343 if (ret != 0) { 2344 rte_errno = -ret; 2345 return i; 2346 } 2347 #endif 2348 2349 /* In case we are supposed to TSO and have DF not set (DF=0) 2350 * hardware must be provided with partial checksum, otherwise 2351 * it will take care of necessary calculations. 2352 */ 2353 2354 ret = rte_net_intel_cksum_flags_prepare(m, 2355 ol_flags & ~PKT_TX_TCP_SEG); 2356 if (ret != 0) { 2357 rte_errno = -ret; 2358 return i; 2359 } 2360 } 2361 2362 return i; 2363 } 2364 2365 static void ena_update_hints(struct ena_adapter *adapter, 2366 struct ena_admin_ena_hw_hints *hints) 2367 { 2368 if (hints->admin_completion_tx_timeout) 2369 adapter->ena_dev.admin_queue.completion_timeout = 2370 hints->admin_completion_tx_timeout * 1000; 2371 2372 if (hints->mmio_read_timeout) 2373 /* convert to usec */ 2374 adapter->ena_dev.mmio_read.reg_read_to = 2375 hints->mmio_read_timeout * 1000; 2376 2377 if (hints->driver_watchdog_timeout) { 2378 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2379 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 2380 else 2381 // Convert msecs to ticks 2382 adapter->keep_alive_timeout = 2383 (hints->driver_watchdog_timeout * 2384 rte_get_timer_hz()) / 1000; 2385 } 2386 } 2387 2388 static int ena_check_space_and_linearize_mbuf(struct ena_ring *tx_ring, 2389 struct rte_mbuf *mbuf) 2390 { 2391 struct ena_com_dev *ena_dev; 2392 int num_segments, header_len, rc; 2393 2394 ena_dev = &tx_ring->adapter->ena_dev; 2395 num_segments = mbuf->nb_segs; 2396 header_len = mbuf->data_len; 2397 2398 if (likely(num_segments < tx_ring->sgl_size)) 2399 goto checkspace; 2400 2401 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && 2402 (num_segments == tx_ring->sgl_size) && 2403 (header_len < tx_ring->tx_max_header_size)) 2404 goto checkspace; 2405 2406 /* Checking for space for 2 additional metadata descriptors due to 2407 * possible header split and metadata descriptor. Linearization will 2408 * be needed so we reduce the segments number from num_segments to 1 2409 */ 2410 if (!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 3)) { 2411 PMD_DRV_LOG(DEBUG, "Not enough space in the tx queue\n"); 2412 return ENA_COM_NO_MEM; 2413 } 2414 ++tx_ring->tx_stats.linearize; 2415 rc = rte_pktmbuf_linearize(mbuf); 2416 if (unlikely(rc)) { 2417 PMD_DRV_LOG(WARNING, "Mbuf linearize failed\n"); 2418 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 2419 ++tx_ring->tx_stats.linearize_failed; 2420 return rc; 2421 } 2422 2423 return 0; 2424 2425 checkspace: 2426 /* Checking for space for 2 additional metadata descriptors due to 2427 * possible header split and metadata descriptor 2428 */ 2429 if (!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 2430 num_segments + 2)) { 2431 PMD_DRV_LOG(DEBUG, "Not enough space in the tx queue\n"); 2432 return ENA_COM_NO_MEM; 2433 } 2434 2435 return 0; 2436 } 2437 2438 static void ena_tx_map_mbuf(struct ena_ring *tx_ring, 2439 struct ena_tx_buffer *tx_info, 2440 struct rte_mbuf *mbuf, 2441 void **push_header, 2442 uint16_t *header_len) 2443 { 2444 struct ena_com_buf *ena_buf; 2445 uint16_t delta, seg_len, push_len; 2446 2447 delta = 0; 2448 seg_len = mbuf->data_len; 2449 2450 tx_info->mbuf = mbuf; 2451 ena_buf = tx_info->bufs; 2452 2453 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2454 /* 2455 * Tx header might be (and will be in most cases) smaller than 2456 * tx_max_header_size. But it's not an issue to send more data 2457 * to the device, than actually needed if the mbuf size is 2458 * greater than tx_max_header_size. 2459 */ 2460 push_len = RTE_MIN(mbuf->pkt_len, tx_ring->tx_max_header_size); 2461 *header_len = push_len; 2462 2463 if (likely(push_len <= seg_len)) { 2464 /* If the push header is in the single segment, then 2465 * just point it to the 1st mbuf data. 2466 */ 2467 *push_header = rte_pktmbuf_mtod(mbuf, uint8_t *); 2468 } else { 2469 /* If the push header lays in the several segments, copy 2470 * it to the intermediate buffer. 2471 */ 2472 rte_pktmbuf_read(mbuf, 0, push_len, 2473 tx_ring->push_buf_intermediate_buf); 2474 *push_header = tx_ring->push_buf_intermediate_buf; 2475 delta = push_len - seg_len; 2476 } 2477 } else { 2478 *push_header = NULL; 2479 *header_len = 0; 2480 push_len = 0; 2481 } 2482 2483 /* Process first segment taking into consideration pushed header */ 2484 if (seg_len > push_len) { 2485 ena_buf->paddr = mbuf->buf_iova + 2486 mbuf->data_off + 2487 push_len; 2488 ena_buf->len = seg_len - push_len; 2489 ena_buf++; 2490 tx_info->num_of_bufs++; 2491 } 2492 2493 while ((mbuf = mbuf->next) != NULL) { 2494 seg_len = mbuf->data_len; 2495 2496 /* Skip mbufs if whole data is pushed as a header */ 2497 if (unlikely(delta > seg_len)) { 2498 delta -= seg_len; 2499 continue; 2500 } 2501 2502 ena_buf->paddr = mbuf->buf_iova + mbuf->data_off + delta; 2503 ena_buf->len = seg_len - delta; 2504 ena_buf++; 2505 tx_info->num_of_bufs++; 2506 2507 delta = 0; 2508 } 2509 } 2510 2511 static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf) 2512 { 2513 struct ena_tx_buffer *tx_info; 2514 struct ena_com_tx_ctx ena_tx_ctx = { { 0 } }; 2515 uint16_t next_to_use; 2516 uint16_t header_len; 2517 uint16_t req_id; 2518 void *push_header; 2519 int nb_hw_desc; 2520 int rc; 2521 2522 rc = ena_check_space_and_linearize_mbuf(tx_ring, mbuf); 2523 if (unlikely(rc)) 2524 return rc; 2525 2526 next_to_use = tx_ring->next_to_use; 2527 2528 req_id = tx_ring->empty_tx_reqs[next_to_use]; 2529 tx_info = &tx_ring->tx_buffer_info[req_id]; 2530 tx_info->num_of_bufs = 0; 2531 2532 ena_tx_map_mbuf(tx_ring, tx_info, mbuf, &push_header, &header_len); 2533 2534 ena_tx_ctx.ena_bufs = tx_info->bufs; 2535 ena_tx_ctx.push_header = push_header; 2536 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 2537 ena_tx_ctx.req_id = req_id; 2538 ena_tx_ctx.header_len = header_len; 2539 2540 /* Set Tx offloads flags, if applicable */ 2541 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads, 2542 tx_ring->disable_meta_caching); 2543 2544 if (unlikely(ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, 2545 &ena_tx_ctx))) { 2546 PMD_DRV_LOG(DEBUG, 2547 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n", 2548 tx_ring->id); 2549 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2550 tx_ring->tx_stats.doorbells++; 2551 tx_ring->pkts_without_db = false; 2552 } 2553 2554 /* prepare the packet's descriptors to dma engine */ 2555 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx, 2556 &nb_hw_desc); 2557 if (unlikely(rc)) { 2558 ++tx_ring->tx_stats.prepare_ctx_err; 2559 return rc; 2560 } 2561 2562 tx_info->tx_descs = nb_hw_desc; 2563 2564 tx_ring->tx_stats.cnt++; 2565 tx_ring->tx_stats.bytes += mbuf->pkt_len; 2566 2567 tx_ring->next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, 2568 tx_ring->size_mask); 2569 2570 return 0; 2571 } 2572 2573 static void ena_tx_cleanup(struct ena_ring *tx_ring) 2574 { 2575 unsigned int cleanup_budget; 2576 unsigned int total_tx_descs = 0; 2577 uint16_t next_to_clean = tx_ring->next_to_clean; 2578 2579 cleanup_budget = RTE_MIN(tx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, 2580 (unsigned int)ENA_REFILL_THRESH_PACKET); 2581 2582 while (likely(total_tx_descs < cleanup_budget)) { 2583 struct rte_mbuf *mbuf; 2584 struct ena_tx_buffer *tx_info; 2585 uint16_t req_id; 2586 2587 if (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) != 0) 2588 break; 2589 2590 if (unlikely(validate_tx_req_id(tx_ring, req_id) != 0)) 2591 break; 2592 2593 /* Get Tx info & store how many descs were processed */ 2594 tx_info = &tx_ring->tx_buffer_info[req_id]; 2595 2596 mbuf = tx_info->mbuf; 2597 rte_pktmbuf_free(mbuf); 2598 2599 tx_info->mbuf = NULL; 2600 tx_ring->empty_tx_reqs[next_to_clean] = req_id; 2601 2602 total_tx_descs += tx_info->tx_descs; 2603 2604 /* Put back descriptor to the ring for reuse */ 2605 next_to_clean = ENA_IDX_NEXT_MASKED(next_to_clean, 2606 tx_ring->size_mask); 2607 } 2608 2609 if (likely(total_tx_descs > 0)) { 2610 /* acknowledge completion of sent packets */ 2611 tx_ring->next_to_clean = next_to_clean; 2612 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 2613 ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); 2614 } 2615 } 2616 2617 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2618 uint16_t nb_pkts) 2619 { 2620 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2621 uint16_t sent_idx = 0; 2622 2623 /* Check adapter state */ 2624 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2625 PMD_DRV_LOG(ALERT, 2626 "Trying to xmit pkts while device is NOT running\n"); 2627 return 0; 2628 } 2629 2630 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 2631 if (ena_xmit_mbuf(tx_ring, tx_pkts[sent_idx])) 2632 break; 2633 tx_ring->pkts_without_db = true; 2634 rte_prefetch0(tx_pkts[ENA_IDX_ADD_MASKED(sent_idx, 4, 2635 tx_ring->size_mask)]); 2636 } 2637 2638 tx_ring->tx_stats.available_desc = 2639 ena_com_free_q_entries(tx_ring->ena_com_io_sq); 2640 2641 /* If there are ready packets to be xmitted... */ 2642 if (likely(tx_ring->pkts_without_db)) { 2643 /* ...let HW do its best :-) */ 2644 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2645 tx_ring->tx_stats.doorbells++; 2646 tx_ring->pkts_without_db = false; 2647 } 2648 2649 ena_tx_cleanup(tx_ring); 2650 2651 tx_ring->tx_stats.available_desc = 2652 ena_com_free_q_entries(tx_ring->ena_com_io_sq); 2653 tx_ring->tx_stats.tx_poll++; 2654 2655 return sent_idx; 2656 } 2657 2658 int ena_copy_eni_stats(struct ena_adapter *adapter) 2659 { 2660 struct ena_admin_eni_stats admin_eni_stats; 2661 int rc; 2662 2663 rte_spinlock_lock(&adapter->admin_lock); 2664 rc = ena_com_get_eni_stats(&adapter->ena_dev, &admin_eni_stats); 2665 rte_spinlock_unlock(&adapter->admin_lock); 2666 if (rc != 0) { 2667 if (rc == ENA_COM_UNSUPPORTED) { 2668 PMD_DRV_LOG(DEBUG, 2669 "Retrieving ENI metrics is not supported.\n"); 2670 } else { 2671 PMD_DRV_LOG(WARNING, 2672 "Failed to get ENI metrics: %d\n", rc); 2673 } 2674 return rc; 2675 } 2676 2677 rte_memcpy(&adapter->eni_stats, &admin_eni_stats, 2678 sizeof(struct ena_stats_eni)); 2679 2680 return 0; 2681 } 2682 2683 /** 2684 * DPDK callback to retrieve names of extended device statistics 2685 * 2686 * @param dev 2687 * Pointer to Ethernet device structure. 2688 * @param[out] xstats_names 2689 * Buffer to insert names into. 2690 * @param n 2691 * Number of names. 2692 * 2693 * @return 2694 * Number of xstats names. 2695 */ 2696 static int ena_xstats_get_names(struct rte_eth_dev *dev, 2697 struct rte_eth_xstat_name *xstats_names, 2698 unsigned int n) 2699 { 2700 unsigned int xstats_count = ena_xstats_calc_num(dev->data); 2701 unsigned int stat, i, count = 0; 2702 2703 if (n < xstats_count || !xstats_names) 2704 return xstats_count; 2705 2706 for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) 2707 strcpy(xstats_names[count].name, 2708 ena_stats_global_strings[stat].name); 2709 2710 for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++) 2711 strcpy(xstats_names[count].name, 2712 ena_stats_eni_strings[stat].name); 2713 2714 for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) 2715 for (i = 0; i < dev->data->nb_rx_queues; i++, count++) 2716 snprintf(xstats_names[count].name, 2717 sizeof(xstats_names[count].name), 2718 "rx_q%d_%s", i, 2719 ena_stats_rx_strings[stat].name); 2720 2721 for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) 2722 for (i = 0; i < dev->data->nb_tx_queues; i++, count++) 2723 snprintf(xstats_names[count].name, 2724 sizeof(xstats_names[count].name), 2725 "tx_q%d_%s", i, 2726 ena_stats_tx_strings[stat].name); 2727 2728 return xstats_count; 2729 } 2730 2731 /** 2732 * DPDK callback to get extended device statistics. 2733 * 2734 * @param dev 2735 * Pointer to Ethernet device structure. 2736 * @param[out] stats 2737 * Stats table output buffer. 2738 * @param n 2739 * The size of the stats table. 2740 * 2741 * @return 2742 * Number of xstats on success, negative on failure. 2743 */ 2744 static int ena_xstats_get(struct rte_eth_dev *dev, 2745 struct rte_eth_xstat *xstats, 2746 unsigned int n) 2747 { 2748 struct ena_adapter *adapter = dev->data->dev_private; 2749 unsigned int xstats_count = ena_xstats_calc_num(dev->data); 2750 unsigned int stat, i, count = 0; 2751 int stat_offset; 2752 void *stats_begin; 2753 2754 if (n < xstats_count) 2755 return xstats_count; 2756 2757 if (!xstats) 2758 return 0; 2759 2760 for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) { 2761 stat_offset = ena_stats_global_strings[stat].stat_offset; 2762 stats_begin = &adapter->dev_stats; 2763 2764 xstats[count].id = count; 2765 xstats[count].value = *((uint64_t *) 2766 ((char *)stats_begin + stat_offset)); 2767 } 2768 2769 /* Even if the function below fails, we should copy previous (or initial 2770 * values) to keep structure of rte_eth_xstat consistent. 2771 */ 2772 ena_copy_eni_stats(adapter); 2773 for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++) { 2774 stat_offset = ena_stats_eni_strings[stat].stat_offset; 2775 stats_begin = &adapter->eni_stats; 2776 2777 xstats[count].id = count; 2778 xstats[count].value = *((uint64_t *) 2779 ((char *)stats_begin + stat_offset)); 2780 } 2781 2782 for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) { 2783 for (i = 0; i < dev->data->nb_rx_queues; i++, count++) { 2784 stat_offset = ena_stats_rx_strings[stat].stat_offset; 2785 stats_begin = &adapter->rx_ring[i].rx_stats; 2786 2787 xstats[count].id = count; 2788 xstats[count].value = *((uint64_t *) 2789 ((char *)stats_begin + stat_offset)); 2790 } 2791 } 2792 2793 for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) { 2794 for (i = 0; i < dev->data->nb_tx_queues; i++, count++) { 2795 stat_offset = ena_stats_tx_strings[stat].stat_offset; 2796 stats_begin = &adapter->tx_ring[i].rx_stats; 2797 2798 xstats[count].id = count; 2799 xstats[count].value = *((uint64_t *) 2800 ((char *)stats_begin + stat_offset)); 2801 } 2802 } 2803 2804 return count; 2805 } 2806 2807 static int ena_xstats_get_by_id(struct rte_eth_dev *dev, 2808 const uint64_t *ids, 2809 uint64_t *values, 2810 unsigned int n) 2811 { 2812 struct ena_adapter *adapter = dev->data->dev_private; 2813 uint64_t id; 2814 uint64_t rx_entries, tx_entries; 2815 unsigned int i; 2816 int qid; 2817 int valid = 0; 2818 bool was_eni_copied = false; 2819 2820 for (i = 0; i < n; ++i) { 2821 id = ids[i]; 2822 /* Check if id belongs to global statistics */ 2823 if (id < ENA_STATS_ARRAY_GLOBAL) { 2824 values[i] = *((uint64_t *)&adapter->dev_stats + id); 2825 ++valid; 2826 continue; 2827 } 2828 2829 /* Check if id belongs to ENI statistics */ 2830 id -= ENA_STATS_ARRAY_GLOBAL; 2831 if (id < ENA_STATS_ARRAY_ENI) { 2832 /* Avoid reading ENI stats multiple times in a single 2833 * function call, as it requires communication with the 2834 * admin queue. 2835 */ 2836 if (!was_eni_copied) { 2837 was_eni_copied = true; 2838 ena_copy_eni_stats(adapter); 2839 } 2840 values[i] = *((uint64_t *)&adapter->eni_stats + id); 2841 ++valid; 2842 continue; 2843 } 2844 2845 /* Check if id belongs to rx queue statistics */ 2846 id -= ENA_STATS_ARRAY_ENI; 2847 rx_entries = ENA_STATS_ARRAY_RX * dev->data->nb_rx_queues; 2848 if (id < rx_entries) { 2849 qid = id % dev->data->nb_rx_queues; 2850 id /= dev->data->nb_rx_queues; 2851 values[i] = *((uint64_t *) 2852 &adapter->rx_ring[qid].rx_stats + id); 2853 ++valid; 2854 continue; 2855 } 2856 /* Check if id belongs to rx queue statistics */ 2857 id -= rx_entries; 2858 tx_entries = ENA_STATS_ARRAY_TX * dev->data->nb_tx_queues; 2859 if (id < tx_entries) { 2860 qid = id % dev->data->nb_tx_queues; 2861 id /= dev->data->nb_tx_queues; 2862 values[i] = *((uint64_t *) 2863 &adapter->tx_ring[qid].tx_stats + id); 2864 ++valid; 2865 continue; 2866 } 2867 } 2868 2869 return valid; 2870 } 2871 2872 static int ena_process_bool_devarg(const char *key, 2873 const char *value, 2874 void *opaque) 2875 { 2876 struct ena_adapter *adapter = opaque; 2877 bool bool_value; 2878 2879 /* Parse the value. */ 2880 if (strcmp(value, "1") == 0) { 2881 bool_value = true; 2882 } else if (strcmp(value, "0") == 0) { 2883 bool_value = false; 2884 } else { 2885 PMD_INIT_LOG(ERR, 2886 "Invalid value: '%s' for key '%s'. Accepted: '0' or '1'\n", 2887 value, key); 2888 return -EINVAL; 2889 } 2890 2891 /* Now, assign it to the proper adapter field. */ 2892 if (strcmp(key, ENA_DEVARG_LARGE_LLQ_HDR) == 0) 2893 adapter->use_large_llq_hdr = bool_value; 2894 2895 return 0; 2896 } 2897 2898 static int ena_parse_devargs(struct ena_adapter *adapter, 2899 struct rte_devargs *devargs) 2900 { 2901 static const char * const allowed_args[] = { 2902 ENA_DEVARG_LARGE_LLQ_HDR, 2903 NULL, 2904 }; 2905 struct rte_kvargs *kvlist; 2906 int rc; 2907 2908 if (devargs == NULL) 2909 return 0; 2910 2911 kvlist = rte_kvargs_parse(devargs->args, allowed_args); 2912 if (kvlist == NULL) { 2913 PMD_INIT_LOG(ERR, "Invalid device arguments: %s\n", 2914 devargs->args); 2915 return -EINVAL; 2916 } 2917 2918 rc = rte_kvargs_process(kvlist, ENA_DEVARG_LARGE_LLQ_HDR, 2919 ena_process_bool_devarg, adapter); 2920 2921 rte_kvargs_free(kvlist); 2922 2923 return rc; 2924 } 2925 2926 /********************************************************************* 2927 * PMD configuration 2928 *********************************************************************/ 2929 static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2930 struct rte_pci_device *pci_dev) 2931 { 2932 return rte_eth_dev_pci_generic_probe(pci_dev, 2933 sizeof(struct ena_adapter), eth_ena_dev_init); 2934 } 2935 2936 static int eth_ena_pci_remove(struct rte_pci_device *pci_dev) 2937 { 2938 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ena_dev_uninit); 2939 } 2940 2941 static struct rte_pci_driver rte_ena_pmd = { 2942 .id_table = pci_id_ena_map, 2943 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 2944 RTE_PCI_DRV_WC_ACTIVATE, 2945 .probe = eth_ena_pci_probe, 2946 .remove = eth_ena_pci_remove, 2947 }; 2948 2949 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd); 2950 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 2951 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci"); 2952 RTE_PMD_REGISTER_PARAM_STRING(net_ena, ENA_DEVARG_LARGE_LLQ_HDR "=<0|1>"); 2953 RTE_LOG_REGISTER_SUFFIX(ena_logtype_init, init, NOTICE); 2954 RTE_LOG_REGISTER_SUFFIX(ena_logtype_driver, driver, NOTICE); 2955 #ifdef RTE_LIBRTE_ENA_DEBUG_RX 2956 RTE_LOG_REGISTER_SUFFIX(ena_logtype_rx, rx, NOTICE); 2957 #endif 2958 #ifdef RTE_LIBRTE_ENA_DEBUG_TX 2959 RTE_LOG_REGISTER_SUFFIX(ena_logtype_tx, tx, NOTICE); 2960 #endif 2961 #ifdef RTE_LIBRTE_ENA_DEBUG_TX_FREE 2962 RTE_LOG_REGISTER_SUFFIX(ena_logtype_tx_free, tx_free, NOTICE); 2963 #endif 2964 #ifdef RTE_LIBRTE_ENA_COM_DEBUG 2965 RTE_LOG_REGISTER_SUFFIX(ena_logtype_com, com, NOTICE); 2966 #endif 2967 2968 /****************************************************************************** 2969 ******************************** AENQ Handlers ******************************* 2970 *****************************************************************************/ 2971 static void ena_update_on_link_change(void *adapter_data, 2972 struct ena_admin_aenq_entry *aenq_e) 2973 { 2974 struct rte_eth_dev *eth_dev = adapter_data; 2975 struct ena_adapter *adapter = eth_dev->data->dev_private; 2976 struct ena_admin_aenq_link_change_desc *aenq_link_desc; 2977 uint32_t status; 2978 2979 aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 2980 2981 status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc); 2982 adapter->link_status = status; 2983 2984 ena_link_update(eth_dev, 0); 2985 rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2986 } 2987 2988 static void ena_notification(void *adapter_data, 2989 struct ena_admin_aenq_entry *aenq_e) 2990 { 2991 struct rte_eth_dev *eth_dev = adapter_data; 2992 struct ena_adapter *adapter = eth_dev->data->dev_private; 2993 struct ena_admin_ena_hw_hints *hints; 2994 2995 if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION) 2996 PMD_DRV_LOG(WARNING, "Invalid group(%x) expected %x\n", 2997 aenq_e->aenq_common_desc.group, 2998 ENA_ADMIN_NOTIFICATION); 2999 3000 switch (aenq_e->aenq_common_desc.syndrome) { 3001 case ENA_ADMIN_UPDATE_HINTS: 3002 hints = (struct ena_admin_ena_hw_hints *) 3003 (&aenq_e->inline_data_w4); 3004 ena_update_hints(adapter, hints); 3005 break; 3006 default: 3007 PMD_DRV_LOG(ERR, "Invalid aenq notification link state %d\n", 3008 aenq_e->aenq_common_desc.syndrome); 3009 } 3010 } 3011 3012 static void ena_keep_alive(void *adapter_data, 3013 __rte_unused struct ena_admin_aenq_entry *aenq_e) 3014 { 3015 struct rte_eth_dev *eth_dev = adapter_data; 3016 struct ena_adapter *adapter = eth_dev->data->dev_private; 3017 struct ena_admin_aenq_keep_alive_desc *desc; 3018 uint64_t rx_drops; 3019 uint64_t tx_drops; 3020 3021 adapter->timestamp_wd = rte_get_timer_cycles(); 3022 3023 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 3024 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 3025 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low; 3026 3027 adapter->drv_stats->rx_drops = rx_drops; 3028 adapter->dev_stats.tx_drops = tx_drops; 3029 } 3030 3031 /** 3032 * This handler will called for unknown event group or unimplemented handlers 3033 **/ 3034 static void unimplemented_aenq_handler(__rte_unused void *data, 3035 __rte_unused struct ena_admin_aenq_entry *aenq_e) 3036 { 3037 PMD_DRV_LOG(ERR, "Unknown event was received or event with " 3038 "unimplemented handler\n"); 3039 } 3040 3041 static struct ena_aenq_handlers aenq_handlers = { 3042 .handlers = { 3043 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 3044 [ENA_ADMIN_NOTIFICATION] = ena_notification, 3045 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive 3046 }, 3047 .unimplemented_handler = unimplemented_aenq_handler 3048 }; 3049