1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. 3 * All rights reserved. 4 */ 5 6 #include <rte_string_fns.h> 7 #include <rte_ether.h> 8 #include <rte_ethdev_driver.h> 9 #include <rte_ethdev_pci.h> 10 #include <rte_tcp.h> 11 #include <rte_atomic.h> 12 #include <rte_dev.h> 13 #include <rte_errno.h> 14 #include <rte_version.h> 15 #include <rte_net.h> 16 17 #include "ena_ethdev.h" 18 #include "ena_logs.h" 19 #include "ena_platform.h" 20 #include "ena_com.h" 21 #include "ena_eth_com.h" 22 23 #include <ena_common_defs.h> 24 #include <ena_regs_defs.h> 25 #include <ena_admin_defs.h> 26 #include <ena_eth_io_defs.h> 27 28 #define DRV_MODULE_VER_MAJOR 2 29 #define DRV_MODULE_VER_MINOR 0 30 #define DRV_MODULE_VER_SUBMINOR 2 31 32 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 33 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 34 /*reverse version of ENA_IO_RXQ_IDX*/ 35 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 36 37 /* While processing submitted and completed descriptors (rx and tx path 38 * respectively) in a loop it is desired to: 39 * - perform batch submissions while populating sumbissmion queue 40 * - avoid blocking transmission of other packets during cleanup phase 41 * Hence the utilization ratio of 1/8 of a queue size. 42 */ 43 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 44 45 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 46 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 47 48 #define GET_L4_HDR_LEN(mbuf) \ 49 ((rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, \ 50 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 51 52 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 53 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 54 #define ENA_HASH_KEY_SIZE 40 55 #define ETH_GSTRING_LEN 32 56 57 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 58 59 #define ENA_MIN_RING_DESC 128 60 61 enum ethtool_stringset { 62 ETH_SS_TEST = 0, 63 ETH_SS_STATS, 64 }; 65 66 struct ena_stats { 67 char name[ETH_GSTRING_LEN]; 68 int stat_offset; 69 }; 70 71 #define ENA_STAT_ENTRY(stat, stat_type) { \ 72 .name = #stat, \ 73 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 74 } 75 76 #define ENA_STAT_RX_ENTRY(stat) \ 77 ENA_STAT_ENTRY(stat, rx) 78 79 #define ENA_STAT_TX_ENTRY(stat) \ 80 ENA_STAT_ENTRY(stat, tx) 81 82 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 83 ENA_STAT_ENTRY(stat, dev) 84 85 #define ENA_MAX_RING_SIZE_RX 8192 86 #define ENA_MAX_RING_SIZE_TX 1024 87 88 /* 89 * Each rte_memzone should have unique name. 90 * To satisfy it, count number of allocation and add it to name. 91 */ 92 uint32_t ena_alloc_cnt; 93 94 static const struct ena_stats ena_stats_global_strings[] = { 95 ENA_STAT_GLOBAL_ENTRY(wd_expired), 96 ENA_STAT_GLOBAL_ENTRY(dev_start), 97 ENA_STAT_GLOBAL_ENTRY(dev_stop), 98 }; 99 100 static const struct ena_stats ena_stats_tx_strings[] = { 101 ENA_STAT_TX_ENTRY(cnt), 102 ENA_STAT_TX_ENTRY(bytes), 103 ENA_STAT_TX_ENTRY(prepare_ctx_err), 104 ENA_STAT_TX_ENTRY(linearize), 105 ENA_STAT_TX_ENTRY(linearize_failed), 106 ENA_STAT_TX_ENTRY(tx_poll), 107 ENA_STAT_TX_ENTRY(doorbells), 108 ENA_STAT_TX_ENTRY(bad_req_id), 109 ENA_STAT_TX_ENTRY(available_desc), 110 }; 111 112 static const struct ena_stats ena_stats_rx_strings[] = { 113 ENA_STAT_RX_ENTRY(cnt), 114 ENA_STAT_RX_ENTRY(bytes), 115 ENA_STAT_RX_ENTRY(refill_partial), 116 ENA_STAT_RX_ENTRY(bad_csum), 117 ENA_STAT_RX_ENTRY(mbuf_alloc_fail), 118 ENA_STAT_RX_ENTRY(bad_desc_num), 119 ENA_STAT_RX_ENTRY(bad_req_id), 120 }; 121 122 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 123 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 124 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 125 126 #define QUEUE_OFFLOADS (DEV_TX_OFFLOAD_TCP_CKSUM |\ 127 DEV_TX_OFFLOAD_UDP_CKSUM |\ 128 DEV_TX_OFFLOAD_IPV4_CKSUM |\ 129 DEV_TX_OFFLOAD_TCP_TSO) 130 #define MBUF_OFFLOADS (PKT_TX_L4_MASK |\ 131 PKT_TX_IP_CKSUM |\ 132 PKT_TX_TCP_SEG) 133 134 /** Vendor ID used by Amazon devices */ 135 #define PCI_VENDOR_ID_AMAZON 0x1D0F 136 /** Amazon devices */ 137 #define PCI_DEVICE_ID_ENA_VF 0xEC20 138 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 139 140 #define ENA_TX_OFFLOAD_MASK (\ 141 PKT_TX_L4_MASK | \ 142 PKT_TX_IPV6 | \ 143 PKT_TX_IPV4 | \ 144 PKT_TX_IP_CKSUM | \ 145 PKT_TX_TCP_SEG) 146 147 #define ENA_TX_OFFLOAD_NOTSUP_MASK \ 148 (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) 149 150 int ena_logtype_init; 151 int ena_logtype_driver; 152 153 #ifdef RTE_LIBRTE_ENA_DEBUG_RX 154 int ena_logtype_rx; 155 #endif 156 #ifdef RTE_LIBRTE_ENA_DEBUG_TX 157 int ena_logtype_tx; 158 #endif 159 #ifdef RTE_LIBRTE_ENA_DEBUG_TX_FREE 160 int ena_logtype_tx_free; 161 #endif 162 #ifdef RTE_LIBRTE_ENA_COM_DEBUG 163 int ena_logtype_com; 164 #endif 165 166 static const struct rte_pci_id pci_id_ena_map[] = { 167 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 168 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, 169 { .device_id = 0 }, 170 }; 171 172 static struct ena_aenq_handlers aenq_handlers; 173 174 static int ena_device_init(struct ena_com_dev *ena_dev, 175 struct ena_com_dev_get_features_ctx *get_feat_ctx, 176 bool *wd_state); 177 static int ena_dev_configure(struct rte_eth_dev *dev); 178 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 179 uint16_t nb_pkts); 180 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 181 uint16_t nb_pkts); 182 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 183 uint16_t nb_desc, unsigned int socket_id, 184 const struct rte_eth_txconf *tx_conf); 185 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 186 uint16_t nb_desc, unsigned int socket_id, 187 const struct rte_eth_rxconf *rx_conf, 188 struct rte_mempool *mp); 189 static uint16_t eth_ena_recv_pkts(void *rx_queue, 190 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 191 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 192 static void ena_init_rings(struct ena_adapter *adapter); 193 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 194 static int ena_start(struct rte_eth_dev *dev); 195 static void ena_stop(struct rte_eth_dev *dev); 196 static void ena_close(struct rte_eth_dev *dev); 197 static int ena_dev_reset(struct rte_eth_dev *dev); 198 static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 199 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 200 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 201 static void ena_rx_queue_release(void *queue); 202 static void ena_tx_queue_release(void *queue); 203 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 204 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 205 static int ena_link_update(struct rte_eth_dev *dev, 206 int wait_to_complete); 207 static int ena_create_io_queue(struct ena_ring *ring); 208 static void ena_queue_stop(struct ena_ring *ring); 209 static void ena_queue_stop_all(struct rte_eth_dev *dev, 210 enum ena_ring_type ring_type); 211 static int ena_queue_start(struct ena_ring *ring); 212 static int ena_queue_start_all(struct rte_eth_dev *dev, 213 enum ena_ring_type ring_type); 214 static void ena_stats_restart(struct rte_eth_dev *dev); 215 static int ena_infos_get(struct rte_eth_dev *dev, 216 struct rte_eth_dev_info *dev_info); 217 static int ena_rss_reta_update(struct rte_eth_dev *dev, 218 struct rte_eth_rss_reta_entry64 *reta_conf, 219 uint16_t reta_size); 220 static int ena_rss_reta_query(struct rte_eth_dev *dev, 221 struct rte_eth_rss_reta_entry64 *reta_conf, 222 uint16_t reta_size); 223 static void ena_interrupt_handler_rte(void *cb_arg); 224 static void ena_timer_wd_callback(struct rte_timer *timer, void *arg); 225 static void ena_destroy_device(struct rte_eth_dev *eth_dev); 226 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev); 227 static int ena_xstats_get_names(struct rte_eth_dev *dev, 228 struct rte_eth_xstat_name *xstats_names, 229 unsigned int n); 230 static int ena_xstats_get(struct rte_eth_dev *dev, 231 struct rte_eth_xstat *stats, 232 unsigned int n); 233 static int ena_xstats_get_by_id(struct rte_eth_dev *dev, 234 const uint64_t *ids, 235 uint64_t *values, 236 unsigned int n); 237 238 static const struct eth_dev_ops ena_dev_ops = { 239 .dev_configure = ena_dev_configure, 240 .dev_infos_get = ena_infos_get, 241 .rx_queue_setup = ena_rx_queue_setup, 242 .tx_queue_setup = ena_tx_queue_setup, 243 .dev_start = ena_start, 244 .dev_stop = ena_stop, 245 .link_update = ena_link_update, 246 .stats_get = ena_stats_get, 247 .xstats_get_names = ena_xstats_get_names, 248 .xstats_get = ena_xstats_get, 249 .xstats_get_by_id = ena_xstats_get_by_id, 250 .mtu_set = ena_mtu_set, 251 .rx_queue_release = ena_rx_queue_release, 252 .tx_queue_release = ena_tx_queue_release, 253 .dev_close = ena_close, 254 .dev_reset = ena_dev_reset, 255 .reta_update = ena_rss_reta_update, 256 .reta_query = ena_rss_reta_query, 257 }; 258 259 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 260 struct ena_com_rx_ctx *ena_rx_ctx) 261 { 262 uint64_t ol_flags = 0; 263 uint32_t packet_type = 0; 264 265 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 266 packet_type |= RTE_PTYPE_L4_TCP; 267 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 268 packet_type |= RTE_PTYPE_L4_UDP; 269 270 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 271 packet_type |= RTE_PTYPE_L3_IPV4; 272 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 273 packet_type |= RTE_PTYPE_L3_IPV6; 274 275 if (!ena_rx_ctx->l4_csum_checked) 276 ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 277 else 278 if (unlikely(ena_rx_ctx->l4_csum_err) && !ena_rx_ctx->frag) 279 ol_flags |= PKT_RX_L4_CKSUM_BAD; 280 else 281 ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 282 283 if (unlikely(ena_rx_ctx->l3_csum_err)) 284 ol_flags |= PKT_RX_IP_CKSUM_BAD; 285 286 mbuf->ol_flags = ol_flags; 287 mbuf->packet_type = packet_type; 288 } 289 290 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 291 struct ena_com_tx_ctx *ena_tx_ctx, 292 uint64_t queue_offloads) 293 { 294 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 295 296 if ((mbuf->ol_flags & MBUF_OFFLOADS) && 297 (queue_offloads & QUEUE_OFFLOADS)) { 298 /* check if TSO is required */ 299 if ((mbuf->ol_flags & PKT_TX_TCP_SEG) && 300 (queue_offloads & DEV_TX_OFFLOAD_TCP_TSO)) { 301 ena_tx_ctx->tso_enable = true; 302 303 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 304 } 305 306 /* check if L3 checksum is needed */ 307 if ((mbuf->ol_flags & PKT_TX_IP_CKSUM) && 308 (queue_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)) 309 ena_tx_ctx->l3_csum_enable = true; 310 311 if (mbuf->ol_flags & PKT_TX_IPV6) { 312 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 313 } else { 314 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 315 316 /* set don't fragment (DF) flag */ 317 if (mbuf->packet_type & 318 (RTE_PTYPE_L4_NONFRAG 319 | RTE_PTYPE_INNER_L4_NONFRAG)) 320 ena_tx_ctx->df = true; 321 } 322 323 /* check if L4 checksum is needed */ 324 if (((mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) && 325 (queue_offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) { 326 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 327 ena_tx_ctx->l4_csum_enable = true; 328 } else if (((mbuf->ol_flags & PKT_TX_L4_MASK) == 329 PKT_TX_UDP_CKSUM) && 330 (queue_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) { 331 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 332 ena_tx_ctx->l4_csum_enable = true; 333 } else { 334 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 335 ena_tx_ctx->l4_csum_enable = false; 336 } 337 338 ena_meta->mss = mbuf->tso_segsz; 339 ena_meta->l3_hdr_len = mbuf->l3_len; 340 ena_meta->l3_hdr_offset = mbuf->l2_len; 341 342 ena_tx_ctx->meta_valid = true; 343 } else { 344 ena_tx_ctx->meta_valid = false; 345 } 346 } 347 348 static inline int validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id) 349 { 350 if (likely(req_id < rx_ring->ring_size)) 351 return 0; 352 353 PMD_DRV_LOG(ERR, "Invalid rx req_id: %hu\n", req_id); 354 355 rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID; 356 rx_ring->adapter->trigger_reset = true; 357 ++rx_ring->rx_stats.bad_req_id; 358 359 return -EFAULT; 360 } 361 362 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) 363 { 364 struct ena_tx_buffer *tx_info = NULL; 365 366 if (likely(req_id < tx_ring->ring_size)) { 367 tx_info = &tx_ring->tx_buffer_info[req_id]; 368 if (likely(tx_info->mbuf)) 369 return 0; 370 } 371 372 if (tx_info) 373 PMD_DRV_LOG(ERR, "tx_info doesn't have valid mbuf\n"); 374 else 375 PMD_DRV_LOG(ERR, "Invalid req_id: %hu\n", req_id); 376 377 /* Trigger device reset */ 378 ++tx_ring->tx_stats.bad_req_id; 379 tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; 380 tx_ring->adapter->trigger_reset = true; 381 return -EFAULT; 382 } 383 384 static void ena_config_host_info(struct ena_com_dev *ena_dev) 385 { 386 struct ena_admin_host_info *host_info; 387 int rc; 388 389 /* Allocate only the host info */ 390 rc = ena_com_allocate_host_info(ena_dev); 391 if (rc) { 392 PMD_DRV_LOG(ERR, "Cannot allocate host info\n"); 393 return; 394 } 395 396 host_info = ena_dev->host_attr.host_info; 397 398 host_info->os_type = ENA_ADMIN_OS_DPDK; 399 host_info->kernel_ver = RTE_VERSION; 400 strlcpy((char *)host_info->kernel_ver_str, rte_version(), 401 sizeof(host_info->kernel_ver_str)); 402 host_info->os_dist = RTE_VERSION; 403 strlcpy((char *)host_info->os_dist_str, rte_version(), 404 sizeof(host_info->os_dist_str)); 405 host_info->driver_version = 406 (DRV_MODULE_VER_MAJOR) | 407 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 408 (DRV_MODULE_VER_SUBMINOR << 409 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 410 host_info->num_cpus = rte_lcore_count(); 411 412 rc = ena_com_set_host_attributes(ena_dev); 413 if (rc) { 414 if (rc == -ENA_COM_UNSUPPORTED) 415 PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); 416 else 417 PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); 418 419 goto err; 420 } 421 422 return; 423 424 err: 425 ena_com_delete_host_info(ena_dev); 426 } 427 428 /* This function calculates the number of xstats based on the current config */ 429 static unsigned int ena_xstats_calc_num(struct rte_eth_dev *dev) 430 { 431 return ENA_STATS_ARRAY_GLOBAL + 432 (dev->data->nb_tx_queues * ENA_STATS_ARRAY_TX) + 433 (dev->data->nb_rx_queues * ENA_STATS_ARRAY_RX); 434 } 435 436 static void ena_config_debug_area(struct ena_adapter *adapter) 437 { 438 u32 debug_area_size; 439 int rc, ss_count; 440 441 ss_count = ena_xstats_calc_num(adapter->rte_dev); 442 443 /* allocate 32 bytes for each string and 64bit for the value */ 444 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 445 446 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 447 if (rc) { 448 PMD_DRV_LOG(ERR, "Cannot allocate debug area\n"); 449 return; 450 } 451 452 rc = ena_com_set_host_attributes(&adapter->ena_dev); 453 if (rc) { 454 if (rc == -ENA_COM_UNSUPPORTED) 455 PMD_DRV_LOG(WARNING, "Cannot set host attributes\n"); 456 else 457 PMD_DRV_LOG(ERR, "Cannot set host attributes\n"); 458 459 goto err; 460 } 461 462 return; 463 err: 464 ena_com_delete_debug_area(&adapter->ena_dev); 465 } 466 467 static void ena_close(struct rte_eth_dev *dev) 468 { 469 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 470 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 471 struct ena_adapter *adapter = dev->data->dev_private; 472 473 if (adapter->state == ENA_ADAPTER_STATE_RUNNING) 474 ena_stop(dev); 475 adapter->state = ENA_ADAPTER_STATE_CLOSED; 476 477 ena_rx_queue_release_all(dev); 478 ena_tx_queue_release_all(dev); 479 480 rte_free(adapter->drv_stats); 481 adapter->drv_stats = NULL; 482 483 rte_intr_disable(intr_handle); 484 rte_intr_callback_unregister(intr_handle, 485 ena_interrupt_handler_rte, 486 adapter); 487 488 /* 489 * MAC is not allocated dynamically. Setting NULL should prevent from 490 * release of the resource in the rte_eth_dev_release_port(). 491 */ 492 dev->data->mac_addrs = NULL; 493 } 494 495 static int 496 ena_dev_reset(struct rte_eth_dev *dev) 497 { 498 int rc = 0; 499 500 ena_destroy_device(dev); 501 rc = eth_ena_dev_init(dev); 502 if (rc) 503 PMD_INIT_LOG(CRIT, "Cannot initialize device"); 504 505 return rc; 506 } 507 508 static int ena_rss_reta_update(struct rte_eth_dev *dev, 509 struct rte_eth_rss_reta_entry64 *reta_conf, 510 uint16_t reta_size) 511 { 512 struct ena_adapter *adapter = dev->data->dev_private; 513 struct ena_com_dev *ena_dev = &adapter->ena_dev; 514 int rc, i; 515 u16 entry_value; 516 int conf_idx; 517 int idx; 518 519 if ((reta_size == 0) || (reta_conf == NULL)) 520 return -EINVAL; 521 522 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 523 PMD_DRV_LOG(WARNING, 524 "indirection table %d is bigger than supported (%d)\n", 525 reta_size, ENA_RX_RSS_TABLE_SIZE); 526 return -EINVAL; 527 } 528 529 for (i = 0 ; i < reta_size ; i++) { 530 /* each reta_conf is for 64 entries. 531 * to support 128 we use 2 conf of 64 532 */ 533 conf_idx = i / RTE_RETA_GROUP_SIZE; 534 idx = i % RTE_RETA_GROUP_SIZE; 535 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 536 entry_value = 537 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 538 539 rc = ena_com_indirect_table_fill_entry(ena_dev, 540 i, 541 entry_value); 542 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 543 PMD_DRV_LOG(ERR, 544 "Cannot fill indirect table\n"); 545 return rc; 546 } 547 } 548 } 549 550 rc = ena_com_indirect_table_set(ena_dev); 551 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 552 PMD_DRV_LOG(ERR, "Cannot flush the indirect table\n"); 553 return rc; 554 } 555 556 PMD_DRV_LOG(DEBUG, "%s(): RSS configured %d entries for port %d\n", 557 __func__, reta_size, adapter->rte_dev->data->port_id); 558 559 return 0; 560 } 561 562 /* Query redirection table. */ 563 static int ena_rss_reta_query(struct rte_eth_dev *dev, 564 struct rte_eth_rss_reta_entry64 *reta_conf, 565 uint16_t reta_size) 566 { 567 struct ena_adapter *adapter = dev->data->dev_private; 568 struct ena_com_dev *ena_dev = &adapter->ena_dev; 569 int rc; 570 int i; 571 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 572 int reta_conf_idx; 573 int reta_idx; 574 575 if (reta_size == 0 || reta_conf == NULL || 576 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 577 return -EINVAL; 578 579 rc = ena_com_indirect_table_get(ena_dev, indirect_table); 580 if (unlikely(rc && rc != ENA_COM_UNSUPPORTED)) { 581 PMD_DRV_LOG(ERR, "cannot get indirect table\n"); 582 return -ENOTSUP; 583 } 584 585 for (i = 0 ; i < reta_size ; i++) { 586 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 587 reta_idx = i % RTE_RETA_GROUP_SIZE; 588 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 589 reta_conf[reta_conf_idx].reta[reta_idx] = 590 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 591 } 592 593 return 0; 594 } 595 596 static int ena_rss_init_default(struct ena_adapter *adapter) 597 { 598 struct ena_com_dev *ena_dev = &adapter->ena_dev; 599 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 600 int rc, i; 601 u32 val; 602 603 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 604 if (unlikely(rc)) { 605 PMD_DRV_LOG(ERR, "Cannot init indirect table\n"); 606 goto err_rss_init; 607 } 608 609 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 610 val = i % nb_rx_queues; 611 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 612 ENA_IO_RXQ_IDX(val)); 613 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 614 PMD_DRV_LOG(ERR, "Cannot fill indirect table\n"); 615 goto err_fill_indir; 616 } 617 } 618 619 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 620 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 621 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 622 PMD_DRV_LOG(INFO, "Cannot fill hash function\n"); 623 goto err_fill_indir; 624 } 625 626 rc = ena_com_set_default_hash_ctrl(ena_dev); 627 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 628 PMD_DRV_LOG(INFO, "Cannot fill hash control\n"); 629 goto err_fill_indir; 630 } 631 632 rc = ena_com_indirect_table_set(ena_dev); 633 if (unlikely(rc && (rc != ENA_COM_UNSUPPORTED))) { 634 PMD_DRV_LOG(ERR, "Cannot flush the indirect table\n"); 635 goto err_fill_indir; 636 } 637 PMD_DRV_LOG(DEBUG, "RSS configured for port %d\n", 638 adapter->rte_dev->data->port_id); 639 640 return 0; 641 642 err_fill_indir: 643 ena_com_rss_destroy(ena_dev); 644 err_rss_init: 645 646 return rc; 647 } 648 649 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 650 { 651 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 652 int nb_queues = dev->data->nb_rx_queues; 653 int i; 654 655 for (i = 0; i < nb_queues; i++) 656 ena_rx_queue_release(queues[i]); 657 } 658 659 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 660 { 661 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 662 int nb_queues = dev->data->nb_tx_queues; 663 int i; 664 665 for (i = 0; i < nb_queues; i++) 666 ena_tx_queue_release(queues[i]); 667 } 668 669 static void ena_rx_queue_release(void *queue) 670 { 671 struct ena_ring *ring = (struct ena_ring *)queue; 672 673 /* Free ring resources */ 674 if (ring->rx_buffer_info) 675 rte_free(ring->rx_buffer_info); 676 ring->rx_buffer_info = NULL; 677 678 if (ring->rx_refill_buffer) 679 rte_free(ring->rx_refill_buffer); 680 ring->rx_refill_buffer = NULL; 681 682 if (ring->empty_rx_reqs) 683 rte_free(ring->empty_rx_reqs); 684 ring->empty_rx_reqs = NULL; 685 686 ring->configured = 0; 687 688 PMD_DRV_LOG(NOTICE, "RX Queue %d:%d released\n", 689 ring->port_id, ring->id); 690 } 691 692 static void ena_tx_queue_release(void *queue) 693 { 694 struct ena_ring *ring = (struct ena_ring *)queue; 695 696 /* Free ring resources */ 697 if (ring->push_buf_intermediate_buf) 698 rte_free(ring->push_buf_intermediate_buf); 699 700 if (ring->tx_buffer_info) 701 rte_free(ring->tx_buffer_info); 702 703 if (ring->empty_tx_reqs) 704 rte_free(ring->empty_tx_reqs); 705 706 ring->empty_tx_reqs = NULL; 707 ring->tx_buffer_info = NULL; 708 ring->push_buf_intermediate_buf = NULL; 709 710 ring->configured = 0; 711 712 PMD_DRV_LOG(NOTICE, "TX Queue %d:%d released\n", 713 ring->port_id, ring->id); 714 } 715 716 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 717 { 718 unsigned int i; 719 720 for (i = 0; i < ring->ring_size; ++i) 721 if (ring->rx_buffer_info[i]) { 722 rte_mbuf_raw_free(ring->rx_buffer_info[i]); 723 ring->rx_buffer_info[i] = NULL; 724 } 725 } 726 727 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 728 { 729 unsigned int i; 730 731 for (i = 0; i < ring->ring_size; ++i) { 732 struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; 733 734 if (tx_buf->mbuf) 735 rte_pktmbuf_free(tx_buf->mbuf); 736 } 737 } 738 739 static int ena_link_update(struct rte_eth_dev *dev, 740 __rte_unused int wait_to_complete) 741 { 742 struct rte_eth_link *link = &dev->data->dev_link; 743 struct ena_adapter *adapter = dev->data->dev_private; 744 745 link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN; 746 link->link_speed = ETH_SPEED_NUM_NONE; 747 link->link_duplex = ETH_LINK_FULL_DUPLEX; 748 749 return 0; 750 } 751 752 static int ena_queue_start_all(struct rte_eth_dev *dev, 753 enum ena_ring_type ring_type) 754 { 755 struct ena_adapter *adapter = dev->data->dev_private; 756 struct ena_ring *queues = NULL; 757 int nb_queues; 758 int i = 0; 759 int rc = 0; 760 761 if (ring_type == ENA_RING_TYPE_RX) { 762 queues = adapter->rx_ring; 763 nb_queues = dev->data->nb_rx_queues; 764 } else { 765 queues = adapter->tx_ring; 766 nb_queues = dev->data->nb_tx_queues; 767 } 768 for (i = 0; i < nb_queues; i++) { 769 if (queues[i].configured) { 770 if (ring_type == ENA_RING_TYPE_RX) { 771 ena_assert_msg( 772 dev->data->rx_queues[i] == &queues[i], 773 "Inconsistent state of rx queues\n"); 774 } else { 775 ena_assert_msg( 776 dev->data->tx_queues[i] == &queues[i], 777 "Inconsistent state of tx queues\n"); 778 } 779 780 rc = ena_queue_start(&queues[i]); 781 782 if (rc) { 783 PMD_INIT_LOG(ERR, 784 "failed to start queue %d type(%d)", 785 i, ring_type); 786 goto err; 787 } 788 } 789 } 790 791 return 0; 792 793 err: 794 while (i--) 795 if (queues[i].configured) 796 ena_queue_stop(&queues[i]); 797 798 return rc; 799 } 800 801 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 802 { 803 uint32_t max_frame_len = adapter->max_mtu; 804 805 if (adapter->rte_eth_dev_data->dev_conf.rxmode.offloads & 806 DEV_RX_OFFLOAD_JUMBO_FRAME) 807 max_frame_len = 808 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 809 810 return max_frame_len; 811 } 812 813 static int ena_check_valid_conf(struct ena_adapter *adapter) 814 { 815 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 816 817 if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) { 818 PMD_INIT_LOG(ERR, "Unsupported MTU of %d. " 819 "max mtu: %d, min mtu: %d", 820 max_frame_len, adapter->max_mtu, ENA_MIN_MTU); 821 return ENA_COM_UNSUPPORTED; 822 } 823 824 return 0; 825 } 826 827 static int 828 ena_calc_queue_size(struct ena_calc_queue_size_ctx *ctx) 829 { 830 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; 831 struct ena_com_dev *ena_dev = ctx->ena_dev; 832 uint32_t tx_queue_size = ENA_MAX_RING_SIZE_TX; 833 uint32_t rx_queue_size = ENA_MAX_RING_SIZE_RX; 834 835 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 836 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 837 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; 838 rx_queue_size = RTE_MIN(rx_queue_size, 839 max_queue_ext->max_rx_cq_depth); 840 rx_queue_size = RTE_MIN(rx_queue_size, 841 max_queue_ext->max_rx_sq_depth); 842 tx_queue_size = RTE_MIN(tx_queue_size, 843 max_queue_ext->max_tx_cq_depth); 844 845 if (ena_dev->tx_mem_queue_type == 846 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 847 tx_queue_size = RTE_MIN(tx_queue_size, 848 llq->max_llq_depth); 849 } else { 850 tx_queue_size = RTE_MIN(tx_queue_size, 851 max_queue_ext->max_tx_sq_depth); 852 } 853 854 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 855 max_queue_ext->max_per_packet_rx_descs); 856 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 857 max_queue_ext->max_per_packet_tx_descs); 858 } else { 859 struct ena_admin_queue_feature_desc *max_queues = 860 &ctx->get_feat_ctx->max_queues; 861 rx_queue_size = RTE_MIN(rx_queue_size, 862 max_queues->max_cq_depth); 863 rx_queue_size = RTE_MIN(rx_queue_size, 864 max_queues->max_sq_depth); 865 tx_queue_size = RTE_MIN(tx_queue_size, 866 max_queues->max_cq_depth); 867 868 if (ena_dev->tx_mem_queue_type == 869 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 870 tx_queue_size = RTE_MIN(tx_queue_size, 871 llq->max_llq_depth); 872 } else { 873 tx_queue_size = RTE_MIN(tx_queue_size, 874 max_queues->max_sq_depth); 875 } 876 877 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 878 max_queues->max_packet_tx_descs); 879 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, 880 max_queues->max_packet_rx_descs); 881 } 882 883 /* Round down to the nearest power of 2 */ 884 rx_queue_size = rte_align32prevpow2(rx_queue_size); 885 tx_queue_size = rte_align32prevpow2(tx_queue_size); 886 887 if (unlikely(rx_queue_size == 0 || tx_queue_size == 0)) { 888 PMD_INIT_LOG(ERR, "Invalid queue size"); 889 return -EFAULT; 890 } 891 892 ctx->rx_queue_size = rx_queue_size; 893 ctx->tx_queue_size = tx_queue_size; 894 895 return 0; 896 } 897 898 static void ena_stats_restart(struct rte_eth_dev *dev) 899 { 900 struct ena_adapter *adapter = dev->data->dev_private; 901 902 rte_atomic64_init(&adapter->drv_stats->ierrors); 903 rte_atomic64_init(&adapter->drv_stats->oerrors); 904 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 905 rte_atomic64_init(&adapter->drv_stats->rx_drops); 906 } 907 908 static int ena_stats_get(struct rte_eth_dev *dev, 909 struct rte_eth_stats *stats) 910 { 911 struct ena_admin_basic_stats ena_stats; 912 struct ena_adapter *adapter = dev->data->dev_private; 913 struct ena_com_dev *ena_dev = &adapter->ena_dev; 914 int rc; 915 int i; 916 int max_rings_stats; 917 918 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 919 return -ENOTSUP; 920 921 memset(&ena_stats, 0, sizeof(ena_stats)); 922 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 923 if (unlikely(rc)) { 924 PMD_DRV_LOG(ERR, "Could not retrieve statistics from ENA\n"); 925 return rc; 926 } 927 928 /* Set of basic statistics from ENA */ 929 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 930 ena_stats.rx_pkts_low); 931 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 932 ena_stats.tx_pkts_low); 933 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 934 ena_stats.rx_bytes_low); 935 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 936 ena_stats.tx_bytes_low); 937 938 /* Driver related stats */ 939 stats->imissed = rte_atomic64_read(&adapter->drv_stats->rx_drops); 940 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 941 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 942 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 943 944 max_rings_stats = RTE_MIN(dev->data->nb_rx_queues, 945 RTE_ETHDEV_QUEUE_STAT_CNTRS); 946 for (i = 0; i < max_rings_stats; ++i) { 947 struct ena_stats_rx *rx_stats = &adapter->rx_ring[i].rx_stats; 948 949 stats->q_ibytes[i] = rx_stats->bytes; 950 stats->q_ipackets[i] = rx_stats->cnt; 951 stats->q_errors[i] = rx_stats->bad_desc_num + 952 rx_stats->bad_req_id; 953 } 954 955 max_rings_stats = RTE_MIN(dev->data->nb_tx_queues, 956 RTE_ETHDEV_QUEUE_STAT_CNTRS); 957 for (i = 0; i < max_rings_stats; ++i) { 958 struct ena_stats_tx *tx_stats = &adapter->tx_ring[i].tx_stats; 959 960 stats->q_obytes[i] = tx_stats->bytes; 961 stats->q_opackets[i] = tx_stats->cnt; 962 } 963 964 return 0; 965 } 966 967 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 968 { 969 struct ena_adapter *adapter; 970 struct ena_com_dev *ena_dev; 971 int rc = 0; 972 973 ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); 974 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); 975 adapter = dev->data->dev_private; 976 977 ena_dev = &adapter->ena_dev; 978 ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); 979 980 if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) { 981 PMD_DRV_LOG(ERR, 982 "Invalid MTU setting. new_mtu: %d " 983 "max mtu: %d min mtu: %d\n", 984 mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU); 985 return -EINVAL; 986 } 987 988 rc = ena_com_set_dev_mtu(ena_dev, mtu); 989 if (rc) 990 PMD_DRV_LOG(ERR, "Could not set MTU: %d\n", mtu); 991 else 992 PMD_DRV_LOG(NOTICE, "Set MTU: %d\n", mtu); 993 994 return rc; 995 } 996 997 static int ena_start(struct rte_eth_dev *dev) 998 { 999 struct ena_adapter *adapter = dev->data->dev_private; 1000 uint64_t ticks; 1001 int rc = 0; 1002 1003 rc = ena_check_valid_conf(adapter); 1004 if (rc) 1005 return rc; 1006 1007 rc = ena_queue_start_all(dev, ENA_RING_TYPE_RX); 1008 if (rc) 1009 return rc; 1010 1011 rc = ena_queue_start_all(dev, ENA_RING_TYPE_TX); 1012 if (rc) 1013 goto err_start_tx; 1014 1015 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 1016 ETH_MQ_RX_RSS_FLAG && adapter->rte_dev->data->nb_rx_queues > 0) { 1017 rc = ena_rss_init_default(adapter); 1018 if (rc) 1019 goto err_rss_init; 1020 } 1021 1022 ena_stats_restart(dev); 1023 1024 adapter->timestamp_wd = rte_get_timer_cycles(); 1025 adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; 1026 1027 ticks = rte_get_timer_hz(); 1028 rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(), 1029 ena_timer_wd_callback, adapter); 1030 1031 ++adapter->dev_stats.dev_start; 1032 adapter->state = ENA_ADAPTER_STATE_RUNNING; 1033 1034 return 0; 1035 1036 err_rss_init: 1037 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1038 err_start_tx: 1039 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1040 return rc; 1041 } 1042 1043 static void ena_stop(struct rte_eth_dev *dev) 1044 { 1045 struct ena_adapter *adapter = dev->data->dev_private; 1046 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1047 int rc; 1048 1049 rte_timer_stop_sync(&adapter->timer_wd); 1050 ena_queue_stop_all(dev, ENA_RING_TYPE_TX); 1051 ena_queue_stop_all(dev, ENA_RING_TYPE_RX); 1052 1053 if (adapter->trigger_reset) { 1054 rc = ena_com_dev_reset(ena_dev, adapter->reset_reason); 1055 if (rc) 1056 PMD_DRV_LOG(ERR, "Device reset failed rc=%d\n", rc); 1057 } 1058 1059 ++adapter->dev_stats.dev_stop; 1060 adapter->state = ENA_ADAPTER_STATE_STOPPED; 1061 } 1062 1063 static int ena_create_io_queue(struct ena_ring *ring) 1064 { 1065 struct ena_adapter *adapter; 1066 struct ena_com_dev *ena_dev; 1067 struct ena_com_create_io_ctx ctx = 1068 /* policy set to _HOST just to satisfy icc compiler */ 1069 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1070 0, 0, 0, 0, 0 }; 1071 uint16_t ena_qid; 1072 unsigned int i; 1073 int rc; 1074 1075 adapter = ring->adapter; 1076 ena_dev = &adapter->ena_dev; 1077 1078 if (ring->type == ENA_RING_TYPE_TX) { 1079 ena_qid = ENA_IO_TXQ_IDX(ring->id); 1080 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1081 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1082 ctx.queue_size = adapter->tx_ring_size; 1083 for (i = 0; i < ring->ring_size; i++) 1084 ring->empty_tx_reqs[i] = i; 1085 } else { 1086 ena_qid = ENA_IO_RXQ_IDX(ring->id); 1087 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1088 ctx.queue_size = adapter->rx_ring_size; 1089 for (i = 0; i < ring->ring_size; i++) 1090 ring->empty_rx_reqs[i] = i; 1091 } 1092 ctx.qid = ena_qid; 1093 ctx.msix_vector = -1; /* interrupts not used */ 1094 ctx.numa_node = ring->numa_socket_id; 1095 1096 rc = ena_com_create_io_queue(ena_dev, &ctx); 1097 if (rc) { 1098 PMD_DRV_LOG(ERR, 1099 "failed to create io queue #%d (qid:%d) rc: %d\n", 1100 ring->id, ena_qid, rc); 1101 return rc; 1102 } 1103 1104 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1105 &ring->ena_com_io_sq, 1106 &ring->ena_com_io_cq); 1107 if (rc) { 1108 PMD_DRV_LOG(ERR, 1109 "Failed to get io queue handlers. queue num %d rc: %d\n", 1110 ring->id, rc); 1111 ena_com_destroy_io_queue(ena_dev, ena_qid); 1112 return rc; 1113 } 1114 1115 if (ring->type == ENA_RING_TYPE_TX) 1116 ena_com_update_numa_node(ring->ena_com_io_cq, ctx.numa_node); 1117 1118 return 0; 1119 } 1120 1121 static void ena_queue_stop(struct ena_ring *ring) 1122 { 1123 struct ena_com_dev *ena_dev = &ring->adapter->ena_dev; 1124 1125 if (ring->type == ENA_RING_TYPE_RX) { 1126 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(ring->id)); 1127 ena_rx_queue_release_bufs(ring); 1128 } else { 1129 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(ring->id)); 1130 ena_tx_queue_release_bufs(ring); 1131 } 1132 } 1133 1134 static void ena_queue_stop_all(struct rte_eth_dev *dev, 1135 enum ena_ring_type ring_type) 1136 { 1137 struct ena_adapter *adapter = dev->data->dev_private; 1138 struct ena_ring *queues = NULL; 1139 uint16_t nb_queues, i; 1140 1141 if (ring_type == ENA_RING_TYPE_RX) { 1142 queues = adapter->rx_ring; 1143 nb_queues = dev->data->nb_rx_queues; 1144 } else { 1145 queues = adapter->tx_ring; 1146 nb_queues = dev->data->nb_tx_queues; 1147 } 1148 1149 for (i = 0; i < nb_queues; ++i) 1150 if (queues[i].configured) 1151 ena_queue_stop(&queues[i]); 1152 } 1153 1154 static int ena_queue_start(struct ena_ring *ring) 1155 { 1156 int rc, bufs_num; 1157 1158 ena_assert_msg(ring->configured == 1, 1159 "Trying to start unconfigured queue\n"); 1160 1161 rc = ena_create_io_queue(ring); 1162 if (rc) { 1163 PMD_INIT_LOG(ERR, "Failed to create IO queue!"); 1164 return rc; 1165 } 1166 1167 ring->next_to_clean = 0; 1168 ring->next_to_use = 0; 1169 1170 if (ring->type == ENA_RING_TYPE_TX) { 1171 ring->tx_stats.available_desc = 1172 ena_com_free_desc(ring->ena_com_io_sq); 1173 return 0; 1174 } 1175 1176 bufs_num = ring->ring_size - 1; 1177 rc = ena_populate_rx_queue(ring, bufs_num); 1178 if (rc != bufs_num) { 1179 ena_com_destroy_io_queue(&ring->adapter->ena_dev, 1180 ENA_IO_RXQ_IDX(ring->id)); 1181 PMD_INIT_LOG(ERR, "Failed to populate rx ring !"); 1182 return ENA_COM_FAULT; 1183 } 1184 1185 return 0; 1186 } 1187 1188 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 1189 uint16_t queue_idx, 1190 uint16_t nb_desc, 1191 unsigned int socket_id, 1192 const struct rte_eth_txconf *tx_conf) 1193 { 1194 struct ena_ring *txq = NULL; 1195 struct ena_adapter *adapter = dev->data->dev_private; 1196 unsigned int i; 1197 1198 txq = &adapter->tx_ring[queue_idx]; 1199 1200 if (txq->configured) { 1201 PMD_DRV_LOG(CRIT, 1202 "API violation. Queue %d is already configured\n", 1203 queue_idx); 1204 return ENA_COM_FAULT; 1205 } 1206 1207 if (!rte_is_power_of_2(nb_desc)) { 1208 PMD_DRV_LOG(ERR, 1209 "Unsupported size of TX queue: %d is not a power of 2.\n", 1210 nb_desc); 1211 return -EINVAL; 1212 } 1213 1214 if (nb_desc > adapter->tx_ring_size) { 1215 PMD_DRV_LOG(ERR, 1216 "Unsupported size of TX queue (max size: %d)\n", 1217 adapter->tx_ring_size); 1218 return -EINVAL; 1219 } 1220 1221 if (nb_desc == RTE_ETH_DEV_FALLBACK_TX_RINGSIZE) 1222 nb_desc = adapter->tx_ring_size; 1223 1224 txq->port_id = dev->data->port_id; 1225 txq->next_to_clean = 0; 1226 txq->next_to_use = 0; 1227 txq->ring_size = nb_desc; 1228 txq->numa_socket_id = socket_id; 1229 1230 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1231 sizeof(struct ena_tx_buffer) * 1232 txq->ring_size, 1233 RTE_CACHE_LINE_SIZE); 1234 if (!txq->tx_buffer_info) { 1235 PMD_DRV_LOG(ERR, "failed to alloc mem for tx buffer info\n"); 1236 return -ENOMEM; 1237 } 1238 1239 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1240 sizeof(u16) * txq->ring_size, 1241 RTE_CACHE_LINE_SIZE); 1242 if (!txq->empty_tx_reqs) { 1243 PMD_DRV_LOG(ERR, "failed to alloc mem for tx reqs\n"); 1244 rte_free(txq->tx_buffer_info); 1245 return -ENOMEM; 1246 } 1247 1248 txq->push_buf_intermediate_buf = 1249 rte_zmalloc("txq->push_buf_intermediate_buf", 1250 txq->tx_max_header_size, 1251 RTE_CACHE_LINE_SIZE); 1252 if (!txq->push_buf_intermediate_buf) { 1253 PMD_DRV_LOG(ERR, "failed to alloc push buff for LLQ\n"); 1254 rte_free(txq->tx_buffer_info); 1255 rte_free(txq->empty_tx_reqs); 1256 return -ENOMEM; 1257 } 1258 1259 for (i = 0; i < txq->ring_size; i++) 1260 txq->empty_tx_reqs[i] = i; 1261 1262 if (tx_conf != NULL) { 1263 txq->offloads = 1264 tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 1265 } 1266 /* Store pointer to this queue in upper layer */ 1267 txq->configured = 1; 1268 dev->data->tx_queues[queue_idx] = txq; 1269 1270 return 0; 1271 } 1272 1273 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1274 uint16_t queue_idx, 1275 uint16_t nb_desc, 1276 unsigned int socket_id, 1277 __rte_unused const struct rte_eth_rxconf *rx_conf, 1278 struct rte_mempool *mp) 1279 { 1280 struct ena_adapter *adapter = dev->data->dev_private; 1281 struct ena_ring *rxq = NULL; 1282 int i; 1283 1284 rxq = &adapter->rx_ring[queue_idx]; 1285 if (rxq->configured) { 1286 PMD_DRV_LOG(CRIT, 1287 "API violation. Queue %d is already configured\n", 1288 queue_idx); 1289 return ENA_COM_FAULT; 1290 } 1291 1292 if (nb_desc == RTE_ETH_DEV_FALLBACK_RX_RINGSIZE) 1293 nb_desc = adapter->rx_ring_size; 1294 1295 if (!rte_is_power_of_2(nb_desc)) { 1296 PMD_DRV_LOG(ERR, 1297 "Unsupported size of RX queue: %d is not a power of 2.\n", 1298 nb_desc); 1299 return -EINVAL; 1300 } 1301 1302 if (nb_desc > adapter->rx_ring_size) { 1303 PMD_DRV_LOG(ERR, 1304 "Unsupported size of RX queue (max size: %d)\n", 1305 adapter->rx_ring_size); 1306 return -EINVAL; 1307 } 1308 1309 rxq->port_id = dev->data->port_id; 1310 rxq->next_to_clean = 0; 1311 rxq->next_to_use = 0; 1312 rxq->ring_size = nb_desc; 1313 rxq->numa_socket_id = socket_id; 1314 rxq->mb_pool = mp; 1315 1316 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1317 sizeof(struct rte_mbuf *) * nb_desc, 1318 RTE_CACHE_LINE_SIZE); 1319 if (!rxq->rx_buffer_info) { 1320 PMD_DRV_LOG(ERR, "failed to alloc mem for rx buffer info\n"); 1321 return -ENOMEM; 1322 } 1323 1324 rxq->rx_refill_buffer = rte_zmalloc("rxq->rx_refill_buffer", 1325 sizeof(struct rte_mbuf *) * nb_desc, 1326 RTE_CACHE_LINE_SIZE); 1327 1328 if (!rxq->rx_refill_buffer) { 1329 PMD_DRV_LOG(ERR, "failed to alloc mem for rx refill buffer\n"); 1330 rte_free(rxq->rx_buffer_info); 1331 rxq->rx_buffer_info = NULL; 1332 return -ENOMEM; 1333 } 1334 1335 rxq->empty_rx_reqs = rte_zmalloc("rxq->empty_rx_reqs", 1336 sizeof(uint16_t) * nb_desc, 1337 RTE_CACHE_LINE_SIZE); 1338 if (!rxq->empty_rx_reqs) { 1339 PMD_DRV_LOG(ERR, "failed to alloc mem for empty rx reqs\n"); 1340 rte_free(rxq->rx_buffer_info); 1341 rxq->rx_buffer_info = NULL; 1342 rte_free(rxq->rx_refill_buffer); 1343 rxq->rx_refill_buffer = NULL; 1344 return -ENOMEM; 1345 } 1346 1347 for (i = 0; i < nb_desc; i++) 1348 rxq->empty_rx_reqs[i] = i; 1349 1350 /* Store pointer to this queue in upper layer */ 1351 rxq->configured = 1; 1352 dev->data->rx_queues[queue_idx] = rxq; 1353 1354 return 0; 1355 } 1356 1357 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1358 { 1359 unsigned int i; 1360 int rc; 1361 uint16_t ring_size = rxq->ring_size; 1362 uint16_t ring_mask = ring_size - 1; 1363 uint16_t next_to_use = rxq->next_to_use; 1364 uint16_t in_use, req_id; 1365 struct rte_mbuf **mbufs = rxq->rx_refill_buffer; 1366 1367 if (unlikely(!count)) 1368 return 0; 1369 1370 in_use = rxq->next_to_use - rxq->next_to_clean; 1371 ena_assert_msg(((in_use + count) < ring_size), "bad ring state\n"); 1372 1373 /* get resources for incoming packets */ 1374 rc = rte_mempool_get_bulk(rxq->mb_pool, (void **)mbufs, count); 1375 if (unlikely(rc < 0)) { 1376 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1377 ++rxq->rx_stats.mbuf_alloc_fail; 1378 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1379 return 0; 1380 } 1381 1382 for (i = 0; i < count; i++) { 1383 uint16_t next_to_use_masked = next_to_use & ring_mask; 1384 struct rte_mbuf *mbuf = mbufs[i]; 1385 struct ena_com_buf ebuf; 1386 1387 if (likely((i + 4) < count)) 1388 rte_prefetch0(mbufs[i + 4]); 1389 1390 req_id = rxq->empty_rx_reqs[next_to_use_masked]; 1391 rc = validate_rx_req_id(rxq, req_id); 1392 if (unlikely(rc < 0)) 1393 break; 1394 rxq->rx_buffer_info[req_id] = mbuf; 1395 1396 /* prepare physical address for DMA transaction */ 1397 ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; 1398 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1399 /* pass resource to device */ 1400 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 1401 &ebuf, req_id); 1402 if (unlikely(rc)) { 1403 PMD_DRV_LOG(WARNING, "failed adding rx desc\n"); 1404 rxq->rx_buffer_info[req_id] = NULL; 1405 break; 1406 } 1407 next_to_use++; 1408 } 1409 1410 if (unlikely(i < count)) { 1411 PMD_DRV_LOG(WARNING, "refilled rx qid %d with only %d " 1412 "buffers (from %d)\n", rxq->id, i, count); 1413 rte_mempool_put_bulk(rxq->mb_pool, (void **)(&mbufs[i]), 1414 count - i); 1415 ++rxq->rx_stats.refill_partial; 1416 } 1417 1418 /* When we submitted free recources to device... */ 1419 if (likely(i > 0)) { 1420 /* ...let HW know that it can fill buffers with data 1421 * 1422 * Add memory barrier to make sure the desc were written before 1423 * issue a doorbell 1424 */ 1425 rte_wmb(); 1426 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1427 1428 rxq->next_to_use = next_to_use; 1429 } 1430 1431 return i; 1432 } 1433 1434 static int ena_device_init(struct ena_com_dev *ena_dev, 1435 struct ena_com_dev_get_features_ctx *get_feat_ctx, 1436 bool *wd_state) 1437 { 1438 uint32_t aenq_groups; 1439 int rc; 1440 bool readless_supported; 1441 1442 /* Initialize mmio registers */ 1443 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1444 if (rc) { 1445 PMD_DRV_LOG(ERR, "failed to init mmio read less\n"); 1446 return rc; 1447 } 1448 1449 /* The PCIe configuration space revision id indicate if mmio reg 1450 * read is disabled. 1451 */ 1452 readless_supported = 1453 !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id 1454 & ENA_MMIO_DISABLE_REG_READ); 1455 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1456 1457 /* reset device */ 1458 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 1459 if (rc) { 1460 PMD_DRV_LOG(ERR, "cannot reset device\n"); 1461 goto err_mmio_read_less; 1462 } 1463 1464 /* check FW version */ 1465 rc = ena_com_validate_version(ena_dev); 1466 if (rc) { 1467 PMD_DRV_LOG(ERR, "device version is too low\n"); 1468 goto err_mmio_read_less; 1469 } 1470 1471 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1472 1473 /* ENA device administration layer init */ 1474 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 1475 if (rc) { 1476 PMD_DRV_LOG(ERR, 1477 "cannot initialize ena admin queue with device\n"); 1478 goto err_mmio_read_less; 1479 } 1480 1481 /* To enable the msix interrupts the driver needs to know the number 1482 * of queues. So the driver uses polling mode to retrieve this 1483 * information. 1484 */ 1485 ena_com_set_admin_polling_mode(ena_dev, true); 1486 1487 ena_config_host_info(ena_dev); 1488 1489 /* Get Device Attributes and features */ 1490 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1491 if (rc) { 1492 PMD_DRV_LOG(ERR, 1493 "cannot get attribute for ena device rc= %d\n", rc); 1494 goto err_admin_init; 1495 } 1496 1497 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 1498 BIT(ENA_ADMIN_NOTIFICATION) | 1499 BIT(ENA_ADMIN_KEEP_ALIVE) | 1500 BIT(ENA_ADMIN_FATAL_ERROR) | 1501 BIT(ENA_ADMIN_WARNING); 1502 1503 aenq_groups &= get_feat_ctx->aenq.supported_groups; 1504 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 1505 if (rc) { 1506 PMD_DRV_LOG(ERR, "Cannot configure aenq groups rc: %d\n", rc); 1507 goto err_admin_init; 1508 } 1509 1510 *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 1511 1512 return 0; 1513 1514 err_admin_init: 1515 ena_com_admin_destroy(ena_dev); 1516 1517 err_mmio_read_less: 1518 ena_com_mmio_reg_read_request_destroy(ena_dev); 1519 1520 return rc; 1521 } 1522 1523 static void ena_interrupt_handler_rte(void *cb_arg) 1524 { 1525 struct ena_adapter *adapter = cb_arg; 1526 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1527 1528 ena_com_admin_q_comp_intr_handler(ena_dev); 1529 if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED)) 1530 ena_com_aenq_intr_handler(ena_dev, adapter); 1531 } 1532 1533 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 1534 { 1535 if (!adapter->wd_state) 1536 return; 1537 1538 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 1539 return; 1540 1541 if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >= 1542 adapter->keep_alive_timeout)) { 1543 PMD_DRV_LOG(ERR, "Keep alive timeout\n"); 1544 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; 1545 adapter->trigger_reset = true; 1546 ++adapter->dev_stats.wd_expired; 1547 } 1548 } 1549 1550 /* Check if admin queue is enabled */ 1551 static void check_for_admin_com_state(struct ena_adapter *adapter) 1552 { 1553 if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) { 1554 PMD_DRV_LOG(ERR, "ENA admin queue is not in running state!\n"); 1555 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; 1556 adapter->trigger_reset = true; 1557 } 1558 } 1559 1560 static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer, 1561 void *arg) 1562 { 1563 struct ena_adapter *adapter = arg; 1564 struct rte_eth_dev *dev = adapter->rte_dev; 1565 1566 check_for_missing_keep_alive(adapter); 1567 check_for_admin_com_state(adapter); 1568 1569 if (unlikely(adapter->trigger_reset)) { 1570 PMD_DRV_LOG(ERR, "Trigger reset is on\n"); 1571 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 1572 NULL); 1573 } 1574 } 1575 1576 static inline void 1577 set_default_llq_configurations(struct ena_llq_configurations *llq_config) 1578 { 1579 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 1580 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B; 1581 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 1582 llq_config->llq_num_decs_before_header = 1583 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 1584 llq_config->llq_ring_entry_size_value = 128; 1585 } 1586 1587 static int 1588 ena_set_queues_placement_policy(struct ena_adapter *adapter, 1589 struct ena_com_dev *ena_dev, 1590 struct ena_admin_feature_llq_desc *llq, 1591 struct ena_llq_configurations *llq_default_configurations) 1592 { 1593 int rc; 1594 u32 llq_feature_mask; 1595 1596 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 1597 if (!(ena_dev->supported_features & llq_feature_mask)) { 1598 PMD_DRV_LOG(INFO, 1599 "LLQ is not supported. Fallback to host mode policy.\n"); 1600 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1601 return 0; 1602 } 1603 1604 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 1605 if (unlikely(rc)) { 1606 PMD_INIT_LOG(WARNING, "Failed to config dev mode. " 1607 "Fallback to host mode policy."); 1608 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1609 return 0; 1610 } 1611 1612 /* Nothing to config, exit */ 1613 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 1614 return 0; 1615 1616 if (!adapter->dev_mem_base) { 1617 PMD_DRV_LOG(ERR, "Unable to access LLQ bar resource. " 1618 "Fallback to host mode policy.\n."); 1619 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1620 return 0; 1621 } 1622 1623 ena_dev->mem_bar = adapter->dev_mem_base; 1624 1625 return 0; 1626 } 1627 1628 static int ena_calc_io_queue_num(struct ena_com_dev *ena_dev, 1629 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1630 { 1631 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, io_queue_num; 1632 1633 /* Regular queues capabilities */ 1634 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 1635 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 1636 &get_feat_ctx->max_queue_ext.max_queue_ext; 1637 io_rx_num = RTE_MIN(max_queue_ext->max_rx_sq_num, 1638 max_queue_ext->max_rx_cq_num); 1639 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 1640 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 1641 } else { 1642 struct ena_admin_queue_feature_desc *max_queues = 1643 &get_feat_ctx->max_queues; 1644 io_tx_sq_num = max_queues->max_sq_num; 1645 io_tx_cq_num = max_queues->max_cq_num; 1646 io_rx_num = RTE_MIN(io_tx_sq_num, io_tx_cq_num); 1647 } 1648 1649 /* In case of LLQ use the llq number in the get feature cmd */ 1650 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 1651 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 1652 1653 io_queue_num = RTE_MIN(ENA_MAX_NUM_IO_QUEUES, io_rx_num); 1654 io_queue_num = RTE_MIN(io_queue_num, io_tx_sq_num); 1655 io_queue_num = RTE_MIN(io_queue_num, io_tx_cq_num); 1656 1657 if (unlikely(io_queue_num == 0)) { 1658 PMD_DRV_LOG(ERR, "Number of IO queues should not be 0\n"); 1659 return -EFAULT; 1660 } 1661 1662 return io_queue_num; 1663 } 1664 1665 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1666 { 1667 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 1668 struct rte_pci_device *pci_dev; 1669 struct rte_intr_handle *intr_handle; 1670 struct ena_adapter *adapter = eth_dev->data->dev_private; 1671 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1672 struct ena_com_dev_get_features_ctx get_feat_ctx; 1673 struct ena_llq_configurations llq_config; 1674 const char *queue_type_str; 1675 int rc; 1676 1677 static int adapters_found; 1678 bool wd_state; 1679 1680 eth_dev->dev_ops = &ena_dev_ops; 1681 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1682 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1683 eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; 1684 1685 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1686 return 0; 1687 1688 memset(adapter, 0, sizeof(struct ena_adapter)); 1689 ena_dev = &adapter->ena_dev; 1690 1691 adapter->rte_eth_dev_data = eth_dev->data; 1692 adapter->rte_dev = eth_dev; 1693 1694 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1695 adapter->pdev = pci_dev; 1696 1697 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", 1698 pci_dev->addr.domain, 1699 pci_dev->addr.bus, 1700 pci_dev->addr.devid, 1701 pci_dev->addr.function); 1702 1703 intr_handle = &pci_dev->intr_handle; 1704 1705 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1706 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1707 1708 if (!adapter->regs) { 1709 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)", 1710 ENA_REGS_BAR); 1711 return -ENXIO; 1712 } 1713 1714 ena_dev->reg_bar = adapter->regs; 1715 ena_dev->dmadev = adapter->pdev; 1716 1717 adapter->id_number = adapters_found; 1718 1719 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1720 adapter->id_number); 1721 1722 /* device specific initialization routine */ 1723 rc = ena_device_init(ena_dev, &get_feat_ctx, &wd_state); 1724 if (rc) { 1725 PMD_INIT_LOG(CRIT, "Failed to init ENA device"); 1726 goto err; 1727 } 1728 adapter->wd_state = wd_state; 1729 1730 set_default_llq_configurations(&llq_config); 1731 rc = ena_set_queues_placement_policy(adapter, ena_dev, 1732 &get_feat_ctx.llq, &llq_config); 1733 if (unlikely(rc)) { 1734 PMD_INIT_LOG(CRIT, "Failed to set placement policy"); 1735 return rc; 1736 } 1737 1738 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 1739 queue_type_str = "Regular"; 1740 else 1741 queue_type_str = "Low latency"; 1742 PMD_DRV_LOG(INFO, "Placement policy: %s\n", queue_type_str); 1743 1744 calc_queue_ctx.ena_dev = ena_dev; 1745 calc_queue_ctx.get_feat_ctx = &get_feat_ctx; 1746 adapter->num_queues = ena_calc_io_queue_num(ena_dev, 1747 &get_feat_ctx); 1748 1749 rc = ena_calc_queue_size(&calc_queue_ctx); 1750 if (unlikely((rc != 0) || (adapter->num_queues <= 0))) { 1751 rc = -EFAULT; 1752 goto err_device_destroy; 1753 } 1754 1755 adapter->tx_ring_size = calc_queue_ctx.tx_queue_size; 1756 adapter->rx_ring_size = calc_queue_ctx.rx_queue_size; 1757 1758 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 1759 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 1760 1761 /* prepare ring structures */ 1762 ena_init_rings(adapter); 1763 1764 ena_config_debug_area(adapter); 1765 1766 /* Set max MTU for this device */ 1767 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1768 1769 /* set device support for offloads */ 1770 adapter->offloads.tso4_supported = (get_feat_ctx.offload.tx & 1771 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0; 1772 adapter->offloads.tx_csum_supported = (get_feat_ctx.offload.tx & 1773 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) != 0; 1774 adapter->offloads.rx_csum_supported = 1775 (get_feat_ctx.offload.rx_supported & 1776 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) != 0; 1777 1778 /* Copy MAC address and point DPDK to it */ 1779 eth_dev->data->mac_addrs = (struct rte_ether_addr *)adapter->mac_addr; 1780 rte_ether_addr_copy((struct rte_ether_addr *) 1781 get_feat_ctx.dev_attr.mac_addr, 1782 (struct rte_ether_addr *)adapter->mac_addr); 1783 1784 /* 1785 * Pass the information to the rte_eth_dev_close() that it should also 1786 * release the private port resources. 1787 */ 1788 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1789 1790 adapter->drv_stats = rte_zmalloc("adapter stats", 1791 sizeof(*adapter->drv_stats), 1792 RTE_CACHE_LINE_SIZE); 1793 if (!adapter->drv_stats) { 1794 PMD_DRV_LOG(ERR, "failed to alloc mem for adapter stats\n"); 1795 rc = -ENOMEM; 1796 goto err_delete_debug_area; 1797 } 1798 1799 rte_intr_callback_register(intr_handle, 1800 ena_interrupt_handler_rte, 1801 adapter); 1802 rte_intr_enable(intr_handle); 1803 ena_com_set_admin_polling_mode(ena_dev, false); 1804 ena_com_admin_aenq_enable(ena_dev); 1805 1806 if (adapters_found == 0) 1807 rte_timer_subsystem_init(); 1808 rte_timer_init(&adapter->timer_wd); 1809 1810 adapters_found++; 1811 adapter->state = ENA_ADAPTER_STATE_INIT; 1812 1813 return 0; 1814 1815 err_delete_debug_area: 1816 ena_com_delete_debug_area(ena_dev); 1817 1818 err_device_destroy: 1819 ena_com_delete_host_info(ena_dev); 1820 ena_com_admin_destroy(ena_dev); 1821 1822 err: 1823 return rc; 1824 } 1825 1826 static void ena_destroy_device(struct rte_eth_dev *eth_dev) 1827 { 1828 struct ena_adapter *adapter = eth_dev->data->dev_private; 1829 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1830 1831 if (adapter->state == ENA_ADAPTER_STATE_FREE) 1832 return; 1833 1834 ena_com_set_admin_running_state(ena_dev, false); 1835 1836 if (adapter->state != ENA_ADAPTER_STATE_CLOSED) 1837 ena_close(eth_dev); 1838 1839 ena_com_delete_debug_area(ena_dev); 1840 ena_com_delete_host_info(ena_dev); 1841 1842 ena_com_abort_admin_commands(ena_dev); 1843 ena_com_wait_for_abort_completion(ena_dev); 1844 ena_com_admin_destroy(ena_dev); 1845 ena_com_mmio_reg_read_request_destroy(ena_dev); 1846 1847 adapter->state = ENA_ADAPTER_STATE_FREE; 1848 } 1849 1850 static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev) 1851 { 1852 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1853 return 0; 1854 1855 ena_destroy_device(eth_dev); 1856 1857 eth_dev->dev_ops = NULL; 1858 eth_dev->rx_pkt_burst = NULL; 1859 eth_dev->tx_pkt_burst = NULL; 1860 eth_dev->tx_pkt_prepare = NULL; 1861 1862 return 0; 1863 } 1864 1865 static int ena_dev_configure(struct rte_eth_dev *dev) 1866 { 1867 struct ena_adapter *adapter = dev->data->dev_private; 1868 1869 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1870 1871 adapter->tx_selected_offloads = dev->data->dev_conf.txmode.offloads; 1872 adapter->rx_selected_offloads = dev->data->dev_conf.rxmode.offloads; 1873 return 0; 1874 } 1875 1876 static void ena_init_rings(struct ena_adapter *adapter) 1877 { 1878 int i; 1879 1880 for (i = 0; i < adapter->num_queues; i++) { 1881 struct ena_ring *ring = &adapter->tx_ring[i]; 1882 1883 ring->configured = 0; 1884 ring->type = ENA_RING_TYPE_TX; 1885 ring->adapter = adapter; 1886 ring->id = i; 1887 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1888 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1889 ring->sgl_size = adapter->max_tx_sgl_size; 1890 } 1891 1892 for (i = 0; i < adapter->num_queues; i++) { 1893 struct ena_ring *ring = &adapter->rx_ring[i]; 1894 1895 ring->configured = 0; 1896 ring->type = ENA_RING_TYPE_RX; 1897 ring->adapter = adapter; 1898 ring->id = i; 1899 ring->sgl_size = adapter->max_rx_sgl_size; 1900 } 1901 } 1902 1903 static int ena_infos_get(struct rte_eth_dev *dev, 1904 struct rte_eth_dev_info *dev_info) 1905 { 1906 struct ena_adapter *adapter; 1907 struct ena_com_dev *ena_dev; 1908 uint64_t rx_feat = 0, tx_feat = 0; 1909 1910 ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); 1911 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); 1912 adapter = dev->data->dev_private; 1913 1914 ena_dev = &adapter->ena_dev; 1915 ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); 1916 1917 dev_info->speed_capa = 1918 ETH_LINK_SPEED_1G | 1919 ETH_LINK_SPEED_2_5G | 1920 ETH_LINK_SPEED_5G | 1921 ETH_LINK_SPEED_10G | 1922 ETH_LINK_SPEED_25G | 1923 ETH_LINK_SPEED_40G | 1924 ETH_LINK_SPEED_50G | 1925 ETH_LINK_SPEED_100G; 1926 1927 /* Set Tx & Rx features available for device */ 1928 if (adapter->offloads.tso4_supported) 1929 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1930 1931 if (adapter->offloads.tx_csum_supported) 1932 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1933 DEV_TX_OFFLOAD_UDP_CKSUM | 1934 DEV_TX_OFFLOAD_TCP_CKSUM; 1935 1936 if (adapter->offloads.rx_csum_supported) 1937 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1938 DEV_RX_OFFLOAD_UDP_CKSUM | 1939 DEV_RX_OFFLOAD_TCP_CKSUM; 1940 1941 rx_feat |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1942 1943 /* Inform framework about available features */ 1944 dev_info->rx_offload_capa = rx_feat; 1945 dev_info->rx_queue_offload_capa = rx_feat; 1946 dev_info->tx_offload_capa = tx_feat; 1947 dev_info->tx_queue_offload_capa = tx_feat; 1948 1949 dev_info->flow_type_rss_offloads = ETH_RSS_IP | ETH_RSS_TCP | 1950 ETH_RSS_UDP; 1951 1952 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1953 dev_info->max_rx_pktlen = adapter->max_mtu; 1954 dev_info->max_mac_addrs = 1; 1955 1956 dev_info->max_rx_queues = adapter->num_queues; 1957 dev_info->max_tx_queues = adapter->num_queues; 1958 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1959 1960 adapter->tx_supported_offloads = tx_feat; 1961 adapter->rx_supported_offloads = rx_feat; 1962 1963 dev_info->rx_desc_lim.nb_max = adapter->rx_ring_size; 1964 dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC; 1965 dev_info->rx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1966 adapter->max_rx_sgl_size); 1967 dev_info->rx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1968 adapter->max_rx_sgl_size); 1969 1970 dev_info->tx_desc_lim.nb_max = adapter->tx_ring_size; 1971 dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC; 1972 dev_info->tx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1973 adapter->max_tx_sgl_size); 1974 dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS, 1975 adapter->max_tx_sgl_size); 1976 1977 return 0; 1978 } 1979 1980 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1981 uint16_t nb_pkts) 1982 { 1983 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1984 unsigned int ring_size = rx_ring->ring_size; 1985 unsigned int ring_mask = ring_size - 1; 1986 uint16_t next_to_clean = rx_ring->next_to_clean; 1987 uint16_t desc_in_use = 0; 1988 uint16_t req_id; 1989 unsigned int recv_idx = 0; 1990 struct rte_mbuf *mbuf = NULL; 1991 struct rte_mbuf *mbuf_head = NULL; 1992 struct rte_mbuf *mbuf_prev = NULL; 1993 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1994 unsigned int completed; 1995 1996 struct ena_com_rx_ctx ena_rx_ctx; 1997 int rc = 0; 1998 1999 /* Check adapter state */ 2000 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2001 PMD_DRV_LOG(ALERT, 2002 "Trying to receive pkts while device is NOT running\n"); 2003 return 0; 2004 } 2005 2006 desc_in_use = rx_ring->next_to_use - next_to_clean; 2007 if (unlikely(nb_pkts > desc_in_use)) 2008 nb_pkts = desc_in_use; 2009 2010 for (completed = 0; completed < nb_pkts; completed++) { 2011 int segments = 0; 2012 2013 ena_rx_ctx.max_bufs = rx_ring->sgl_size; 2014 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 2015 ena_rx_ctx.descs = 0; 2016 /* receive packet context */ 2017 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 2018 rx_ring->ena_com_io_sq, 2019 &ena_rx_ctx); 2020 if (unlikely(rc)) { 2021 PMD_DRV_LOG(ERR, "ena_com_rx_pkt error %d\n", rc); 2022 rx_ring->adapter->reset_reason = 2023 ENA_REGS_RESET_TOO_MANY_RX_DESCS; 2024 rx_ring->adapter->trigger_reset = true; 2025 ++rx_ring->rx_stats.bad_desc_num; 2026 return 0; 2027 } 2028 2029 if (unlikely(ena_rx_ctx.descs == 0)) 2030 break; 2031 2032 while (segments < ena_rx_ctx.descs) { 2033 req_id = ena_rx_ctx.ena_bufs[segments].req_id; 2034 rc = validate_rx_req_id(rx_ring, req_id); 2035 if (unlikely(rc)) { 2036 if (segments != 0) 2037 rte_mbuf_raw_free(mbuf_head); 2038 break; 2039 } 2040 2041 mbuf = rx_buff_info[req_id]; 2042 rx_buff_info[req_id] = NULL; 2043 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 2044 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 2045 mbuf->refcnt = 1; 2046 mbuf->next = NULL; 2047 if (unlikely(segments == 0)) { 2048 mbuf->nb_segs = ena_rx_ctx.descs; 2049 mbuf->port = rx_ring->port_id; 2050 mbuf->pkt_len = 0; 2051 mbuf_head = mbuf; 2052 } else { 2053 /* for multi-segment pkts create mbuf chain */ 2054 mbuf_prev->next = mbuf; 2055 } 2056 mbuf_head->pkt_len += mbuf->data_len; 2057 2058 mbuf_prev = mbuf; 2059 rx_ring->empty_rx_reqs[next_to_clean & ring_mask] = 2060 req_id; 2061 segments++; 2062 next_to_clean++; 2063 } 2064 if (unlikely(rc)) 2065 break; 2066 2067 /* fill mbuf attributes if any */ 2068 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 2069 2070 if (unlikely(mbuf_head->ol_flags & 2071 (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD))) { 2072 rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors); 2073 ++rx_ring->rx_stats.bad_csum; 2074 } 2075 2076 mbuf_head->hash.rss = ena_rx_ctx.hash; 2077 2078 /* pass to DPDK application head mbuf */ 2079 rx_pkts[recv_idx] = mbuf_head; 2080 recv_idx++; 2081 rx_ring->rx_stats.bytes += mbuf_head->pkt_len; 2082 } 2083 2084 rx_ring->rx_stats.cnt += recv_idx; 2085 rx_ring->next_to_clean = next_to_clean; 2086 2087 desc_in_use = desc_in_use - completed + 1; 2088 /* Burst refill to save doorbells, memory barriers, const interval */ 2089 if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) { 2090 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); 2091 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use); 2092 } 2093 2094 return recv_idx; 2095 } 2096 2097 static uint16_t 2098 eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2099 uint16_t nb_pkts) 2100 { 2101 int32_t ret; 2102 uint32_t i; 2103 struct rte_mbuf *m; 2104 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2105 struct rte_ipv4_hdr *ip_hdr; 2106 uint64_t ol_flags; 2107 uint16_t frag_field; 2108 2109 for (i = 0; i != nb_pkts; i++) { 2110 m = tx_pkts[i]; 2111 ol_flags = m->ol_flags; 2112 2113 if (!(ol_flags & PKT_TX_IPV4)) 2114 continue; 2115 2116 /* If there was not L2 header length specified, assume it is 2117 * length of the ethernet header. 2118 */ 2119 if (unlikely(m->l2_len == 0)) 2120 m->l2_len = sizeof(struct rte_ether_hdr); 2121 2122 ip_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, 2123 m->l2_len); 2124 frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); 2125 2126 if ((frag_field & RTE_IPV4_HDR_DF_FLAG) != 0) { 2127 m->packet_type |= RTE_PTYPE_L4_NONFRAG; 2128 2129 /* If IPv4 header has DF flag enabled and TSO support is 2130 * disabled, partial chcecksum should not be calculated. 2131 */ 2132 if (!tx_ring->adapter->offloads.tso4_supported) 2133 continue; 2134 } 2135 2136 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || 2137 (ol_flags & PKT_TX_L4_MASK) == 2138 PKT_TX_SCTP_CKSUM) { 2139 rte_errno = ENOTSUP; 2140 return i; 2141 } 2142 2143 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 2144 ret = rte_validate_tx_offload(m); 2145 if (ret != 0) { 2146 rte_errno = -ret; 2147 return i; 2148 } 2149 #endif 2150 2151 /* In case we are supposed to TSO and have DF not set (DF=0) 2152 * hardware must be provided with partial checksum, otherwise 2153 * it will take care of necessary calculations. 2154 */ 2155 2156 ret = rte_net_intel_cksum_flags_prepare(m, 2157 ol_flags & ~PKT_TX_TCP_SEG); 2158 if (ret != 0) { 2159 rte_errno = -ret; 2160 return i; 2161 } 2162 } 2163 2164 return i; 2165 } 2166 2167 static void ena_update_hints(struct ena_adapter *adapter, 2168 struct ena_admin_ena_hw_hints *hints) 2169 { 2170 if (hints->admin_completion_tx_timeout) 2171 adapter->ena_dev.admin_queue.completion_timeout = 2172 hints->admin_completion_tx_timeout * 1000; 2173 2174 if (hints->mmio_read_timeout) 2175 /* convert to usec */ 2176 adapter->ena_dev.mmio_read.reg_read_to = 2177 hints->mmio_read_timeout * 1000; 2178 2179 if (hints->driver_watchdog_timeout) { 2180 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2181 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 2182 else 2183 // Convert msecs to ticks 2184 adapter->keep_alive_timeout = 2185 (hints->driver_watchdog_timeout * 2186 rte_get_timer_hz()) / 1000; 2187 } 2188 } 2189 2190 static int ena_check_and_linearize_mbuf(struct ena_ring *tx_ring, 2191 struct rte_mbuf *mbuf) 2192 { 2193 struct ena_com_dev *ena_dev; 2194 int num_segments, header_len, rc; 2195 2196 ena_dev = &tx_ring->adapter->ena_dev; 2197 num_segments = mbuf->nb_segs; 2198 header_len = mbuf->data_len; 2199 2200 if (likely(num_segments < tx_ring->sgl_size)) 2201 return 0; 2202 2203 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && 2204 (num_segments == tx_ring->sgl_size) && 2205 (header_len < tx_ring->tx_max_header_size)) 2206 return 0; 2207 2208 ++tx_ring->tx_stats.linearize; 2209 rc = rte_pktmbuf_linearize(mbuf); 2210 if (unlikely(rc)) { 2211 PMD_DRV_LOG(WARNING, "Mbuf linearize failed\n"); 2212 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 2213 ++tx_ring->tx_stats.linearize_failed; 2214 return rc; 2215 } 2216 2217 return rc; 2218 } 2219 2220 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 2221 uint16_t nb_pkts) 2222 { 2223 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 2224 uint16_t next_to_use = tx_ring->next_to_use; 2225 uint16_t next_to_clean = tx_ring->next_to_clean; 2226 struct rte_mbuf *mbuf; 2227 uint16_t seg_len; 2228 unsigned int ring_size = tx_ring->ring_size; 2229 unsigned int ring_mask = ring_size - 1; 2230 struct ena_com_tx_ctx ena_tx_ctx; 2231 struct ena_tx_buffer *tx_info; 2232 struct ena_com_buf *ebuf; 2233 uint16_t rc, req_id, total_tx_descs = 0; 2234 uint16_t sent_idx = 0, empty_tx_reqs; 2235 uint16_t push_len = 0; 2236 uint16_t delta = 0; 2237 int nb_hw_desc; 2238 uint32_t total_length; 2239 2240 /* Check adapter state */ 2241 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 2242 PMD_DRV_LOG(ALERT, 2243 "Trying to xmit pkts while device is NOT running\n"); 2244 return 0; 2245 } 2246 2247 empty_tx_reqs = ring_size - (next_to_use - next_to_clean); 2248 if (nb_pkts > empty_tx_reqs) 2249 nb_pkts = empty_tx_reqs; 2250 2251 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 2252 mbuf = tx_pkts[sent_idx]; 2253 total_length = 0; 2254 2255 rc = ena_check_and_linearize_mbuf(tx_ring, mbuf); 2256 if (unlikely(rc)) 2257 break; 2258 2259 req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; 2260 tx_info = &tx_ring->tx_buffer_info[req_id]; 2261 tx_info->mbuf = mbuf; 2262 tx_info->num_of_bufs = 0; 2263 ebuf = tx_info->bufs; 2264 2265 /* Prepare TX context */ 2266 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 2267 memset(&ena_tx_ctx.ena_meta, 0x0, 2268 sizeof(struct ena_com_tx_meta)); 2269 ena_tx_ctx.ena_bufs = ebuf; 2270 ena_tx_ctx.req_id = req_id; 2271 2272 delta = 0; 2273 seg_len = mbuf->data_len; 2274 2275 if (tx_ring->tx_mem_queue_type == 2276 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2277 push_len = RTE_MIN(mbuf->pkt_len, 2278 tx_ring->tx_max_header_size); 2279 ena_tx_ctx.header_len = push_len; 2280 2281 if (likely(push_len <= seg_len)) { 2282 /* If the push header is in the single segment, 2283 * then just point it to the 1st mbuf data. 2284 */ 2285 ena_tx_ctx.push_header = 2286 rte_pktmbuf_mtod(mbuf, uint8_t *); 2287 } else { 2288 /* If the push header lays in the several 2289 * segments, copy it to the intermediate buffer. 2290 */ 2291 rte_pktmbuf_read(mbuf, 0, push_len, 2292 tx_ring->push_buf_intermediate_buf); 2293 ena_tx_ctx.push_header = 2294 tx_ring->push_buf_intermediate_buf; 2295 delta = push_len - seg_len; 2296 } 2297 } /* there's no else as we take advantage of memset zeroing */ 2298 2299 /* Set TX offloads flags, if applicable */ 2300 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads); 2301 2302 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 2303 2304 /* Process first segment taking into 2305 * consideration pushed header 2306 */ 2307 if (seg_len > push_len) { 2308 ebuf->paddr = mbuf->buf_iova + 2309 mbuf->data_off + 2310 push_len; 2311 ebuf->len = seg_len - push_len; 2312 ebuf++; 2313 tx_info->num_of_bufs++; 2314 } 2315 total_length += mbuf->data_len; 2316 2317 while ((mbuf = mbuf->next) != NULL) { 2318 seg_len = mbuf->data_len; 2319 2320 /* Skip mbufs if whole data is pushed as a header */ 2321 if (unlikely(delta > seg_len)) { 2322 delta -= seg_len; 2323 continue; 2324 } 2325 2326 ebuf->paddr = mbuf->buf_iova + mbuf->data_off + delta; 2327 ebuf->len = seg_len - delta; 2328 total_length += ebuf->len; 2329 ebuf++; 2330 tx_info->num_of_bufs++; 2331 2332 delta = 0; 2333 } 2334 2335 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 2336 2337 if (ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, 2338 &ena_tx_ctx)) { 2339 PMD_DRV_LOG(DEBUG, "llq tx max burst size of queue %d" 2340 " achieved, writing doorbell to send burst\n", 2341 tx_ring->id); 2342 rte_wmb(); 2343 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2344 } 2345 2346 /* prepare the packet's descriptors to dma engine */ 2347 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 2348 &ena_tx_ctx, &nb_hw_desc); 2349 if (unlikely(rc)) { 2350 ++tx_ring->tx_stats.prepare_ctx_err; 2351 break; 2352 } 2353 tx_info->tx_descs = nb_hw_desc; 2354 2355 next_to_use++; 2356 tx_ring->tx_stats.cnt++; 2357 tx_ring->tx_stats.bytes += total_length; 2358 } 2359 tx_ring->tx_stats.available_desc = 2360 ena_com_free_desc(tx_ring->ena_com_io_sq); 2361 2362 /* If there are ready packets to be xmitted... */ 2363 if (sent_idx > 0) { 2364 /* ...let HW do its best :-) */ 2365 rte_wmb(); 2366 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 2367 tx_ring->tx_stats.doorbells++; 2368 tx_ring->next_to_use = next_to_use; 2369 } 2370 2371 /* Clear complete packets */ 2372 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 2373 rc = validate_tx_req_id(tx_ring, req_id); 2374 if (rc) 2375 break; 2376 2377 /* Get Tx info & store how many descs were processed */ 2378 tx_info = &tx_ring->tx_buffer_info[req_id]; 2379 total_tx_descs += tx_info->tx_descs; 2380 2381 /* Free whole mbuf chain */ 2382 mbuf = tx_info->mbuf; 2383 rte_pktmbuf_free(mbuf); 2384 tx_info->mbuf = NULL; 2385 2386 /* Put back descriptor to the ring for reuse */ 2387 tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; 2388 next_to_clean++; 2389 2390 /* If too many descs to clean, leave it for another run */ 2391 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 2392 break; 2393 } 2394 tx_ring->tx_stats.available_desc = 2395 ena_com_free_desc(tx_ring->ena_com_io_sq); 2396 2397 if (total_tx_descs > 0) { 2398 /* acknowledge completion of sent packets */ 2399 tx_ring->next_to_clean = next_to_clean; 2400 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 2401 ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); 2402 } 2403 2404 tx_ring->tx_stats.tx_poll++; 2405 2406 return sent_idx; 2407 } 2408 2409 /** 2410 * DPDK callback to retrieve names of extended device statistics 2411 * 2412 * @param dev 2413 * Pointer to Ethernet device structure. 2414 * @param[out] xstats_names 2415 * Buffer to insert names into. 2416 * @param n 2417 * Number of names. 2418 * 2419 * @return 2420 * Number of xstats names. 2421 */ 2422 static int ena_xstats_get_names(struct rte_eth_dev *dev, 2423 struct rte_eth_xstat_name *xstats_names, 2424 unsigned int n) 2425 { 2426 unsigned int xstats_count = ena_xstats_calc_num(dev); 2427 unsigned int stat, i, count = 0; 2428 2429 if (n < xstats_count || !xstats_names) 2430 return xstats_count; 2431 2432 for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) 2433 strcpy(xstats_names[count].name, 2434 ena_stats_global_strings[stat].name); 2435 2436 for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) 2437 for (i = 0; i < dev->data->nb_rx_queues; i++, count++) 2438 snprintf(xstats_names[count].name, 2439 sizeof(xstats_names[count].name), 2440 "rx_q%d_%s", i, 2441 ena_stats_rx_strings[stat].name); 2442 2443 for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) 2444 for (i = 0; i < dev->data->nb_tx_queues; i++, count++) 2445 snprintf(xstats_names[count].name, 2446 sizeof(xstats_names[count].name), 2447 "tx_q%d_%s", i, 2448 ena_stats_tx_strings[stat].name); 2449 2450 return xstats_count; 2451 } 2452 2453 /** 2454 * DPDK callback to get extended device statistics. 2455 * 2456 * @param dev 2457 * Pointer to Ethernet device structure. 2458 * @param[out] stats 2459 * Stats table output buffer. 2460 * @param n 2461 * The size of the stats table. 2462 * 2463 * @return 2464 * Number of xstats on success, negative on failure. 2465 */ 2466 static int ena_xstats_get(struct rte_eth_dev *dev, 2467 struct rte_eth_xstat *xstats, 2468 unsigned int n) 2469 { 2470 struct ena_adapter *adapter = dev->data->dev_private; 2471 unsigned int xstats_count = ena_xstats_calc_num(dev); 2472 unsigned int stat, i, count = 0; 2473 int stat_offset; 2474 void *stats_begin; 2475 2476 if (n < xstats_count) 2477 return xstats_count; 2478 2479 if (!xstats) 2480 return 0; 2481 2482 for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) { 2483 stat_offset = ena_stats_rx_strings[stat].stat_offset; 2484 stats_begin = &adapter->dev_stats; 2485 2486 xstats[count].id = count; 2487 xstats[count].value = *((uint64_t *) 2488 ((char *)stats_begin + stat_offset)); 2489 } 2490 2491 for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) { 2492 for (i = 0; i < dev->data->nb_rx_queues; i++, count++) { 2493 stat_offset = ena_stats_rx_strings[stat].stat_offset; 2494 stats_begin = &adapter->rx_ring[i].rx_stats; 2495 2496 xstats[count].id = count; 2497 xstats[count].value = *((uint64_t *) 2498 ((char *)stats_begin + stat_offset)); 2499 } 2500 } 2501 2502 for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) { 2503 for (i = 0; i < dev->data->nb_tx_queues; i++, count++) { 2504 stat_offset = ena_stats_tx_strings[stat].stat_offset; 2505 stats_begin = &adapter->tx_ring[i].rx_stats; 2506 2507 xstats[count].id = count; 2508 xstats[count].value = *((uint64_t *) 2509 ((char *)stats_begin + stat_offset)); 2510 } 2511 } 2512 2513 return count; 2514 } 2515 2516 static int ena_xstats_get_by_id(struct rte_eth_dev *dev, 2517 const uint64_t *ids, 2518 uint64_t *values, 2519 unsigned int n) 2520 { 2521 struct ena_adapter *adapter = dev->data->dev_private; 2522 uint64_t id; 2523 uint64_t rx_entries, tx_entries; 2524 unsigned int i; 2525 int qid; 2526 int valid = 0; 2527 for (i = 0; i < n; ++i) { 2528 id = ids[i]; 2529 /* Check if id belongs to global statistics */ 2530 if (id < ENA_STATS_ARRAY_GLOBAL) { 2531 values[i] = *((uint64_t *)&adapter->dev_stats + id); 2532 ++valid; 2533 continue; 2534 } 2535 2536 /* Check if id belongs to rx queue statistics */ 2537 id -= ENA_STATS_ARRAY_GLOBAL; 2538 rx_entries = ENA_STATS_ARRAY_RX * dev->data->nb_rx_queues; 2539 if (id < rx_entries) { 2540 qid = id % dev->data->nb_rx_queues; 2541 id /= dev->data->nb_rx_queues; 2542 values[i] = *((uint64_t *) 2543 &adapter->rx_ring[qid].rx_stats + id); 2544 ++valid; 2545 continue; 2546 } 2547 /* Check if id belongs to rx queue statistics */ 2548 id -= rx_entries; 2549 tx_entries = ENA_STATS_ARRAY_TX * dev->data->nb_tx_queues; 2550 if (id < tx_entries) { 2551 qid = id % dev->data->nb_tx_queues; 2552 id /= dev->data->nb_tx_queues; 2553 values[i] = *((uint64_t *) 2554 &adapter->tx_ring[qid].tx_stats + id); 2555 ++valid; 2556 continue; 2557 } 2558 } 2559 2560 return valid; 2561 } 2562 2563 /********************************************************************* 2564 * PMD configuration 2565 *********************************************************************/ 2566 static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2567 struct rte_pci_device *pci_dev) 2568 { 2569 return rte_eth_dev_pci_generic_probe(pci_dev, 2570 sizeof(struct ena_adapter), eth_ena_dev_init); 2571 } 2572 2573 static int eth_ena_pci_remove(struct rte_pci_device *pci_dev) 2574 { 2575 return rte_eth_dev_pci_generic_remove(pci_dev, eth_ena_dev_uninit); 2576 } 2577 2578 static struct rte_pci_driver rte_ena_pmd = { 2579 .id_table = pci_id_ena_map, 2580 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 2581 RTE_PCI_DRV_WC_ACTIVATE, 2582 .probe = eth_ena_pci_probe, 2583 .remove = eth_ena_pci_remove, 2584 }; 2585 2586 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd); 2587 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 2588 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci"); 2589 2590 RTE_INIT(ena_init_log) 2591 { 2592 ena_logtype_init = rte_log_register("pmd.net.ena.init"); 2593 if (ena_logtype_init >= 0) 2594 rte_log_set_level(ena_logtype_init, RTE_LOG_NOTICE); 2595 ena_logtype_driver = rte_log_register("pmd.net.ena.driver"); 2596 if (ena_logtype_driver >= 0) 2597 rte_log_set_level(ena_logtype_driver, RTE_LOG_NOTICE); 2598 2599 #ifdef RTE_LIBRTE_ENA_DEBUG_RX 2600 ena_logtype_rx = rte_log_register("pmd.net.ena.rx"); 2601 if (ena_logtype_rx >= 0) 2602 rte_log_set_level(ena_logtype_rx, RTE_LOG_NOTICE); 2603 #endif 2604 2605 #ifdef RTE_LIBRTE_ENA_DEBUG_TX 2606 ena_logtype_tx = rte_log_register("pmd.net.ena.tx"); 2607 if (ena_logtype_tx >= 0) 2608 rte_log_set_level(ena_logtype_tx, RTE_LOG_NOTICE); 2609 #endif 2610 2611 #ifdef RTE_LIBRTE_ENA_DEBUG_TX_FREE 2612 ena_logtype_tx_free = rte_log_register("pmd.net.ena.tx_free"); 2613 if (ena_logtype_tx_free >= 0) 2614 rte_log_set_level(ena_logtype_tx_free, RTE_LOG_NOTICE); 2615 #endif 2616 2617 #ifdef RTE_LIBRTE_ENA_COM_DEBUG 2618 ena_logtype_com = rte_log_register("pmd.net.ena.com"); 2619 if (ena_logtype_com >= 0) 2620 rte_log_set_level(ena_logtype_com, RTE_LOG_NOTICE); 2621 #endif 2622 } 2623 2624 /****************************************************************************** 2625 ******************************** AENQ Handlers ******************************* 2626 *****************************************************************************/ 2627 static void ena_update_on_link_change(void *adapter_data, 2628 struct ena_admin_aenq_entry *aenq_e) 2629 { 2630 struct rte_eth_dev *eth_dev; 2631 struct ena_adapter *adapter; 2632 struct ena_admin_aenq_link_change_desc *aenq_link_desc; 2633 uint32_t status; 2634 2635 adapter = adapter_data; 2636 aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 2637 eth_dev = adapter->rte_dev; 2638 2639 status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc); 2640 adapter->link_status = status; 2641 2642 ena_link_update(eth_dev, 0); 2643 _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2644 } 2645 2646 static void ena_notification(void *data, 2647 struct ena_admin_aenq_entry *aenq_e) 2648 { 2649 struct ena_adapter *adapter = data; 2650 struct ena_admin_ena_hw_hints *hints; 2651 2652 if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION) 2653 PMD_DRV_LOG(WARNING, "Invalid group(%x) expected %x\n", 2654 aenq_e->aenq_common_desc.group, 2655 ENA_ADMIN_NOTIFICATION); 2656 2657 switch (aenq_e->aenq_common_desc.syndrom) { 2658 case ENA_ADMIN_UPDATE_HINTS: 2659 hints = (struct ena_admin_ena_hw_hints *) 2660 (&aenq_e->inline_data_w4); 2661 ena_update_hints(adapter, hints); 2662 break; 2663 default: 2664 PMD_DRV_LOG(ERR, "Invalid aenq notification link state %d\n", 2665 aenq_e->aenq_common_desc.syndrom); 2666 } 2667 } 2668 2669 static void ena_keep_alive(void *adapter_data, 2670 __rte_unused struct ena_admin_aenq_entry *aenq_e) 2671 { 2672 struct ena_adapter *adapter = adapter_data; 2673 struct ena_admin_aenq_keep_alive_desc *desc; 2674 uint64_t rx_drops; 2675 2676 adapter->timestamp_wd = rte_get_timer_cycles(); 2677 2678 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 2679 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 2680 rte_atomic64_set(&adapter->drv_stats->rx_drops, rx_drops); 2681 } 2682 2683 /** 2684 * This handler will called for unknown event group or unimplemented handlers 2685 **/ 2686 static void unimplemented_aenq_handler(__rte_unused void *data, 2687 __rte_unused struct ena_admin_aenq_entry *aenq_e) 2688 { 2689 PMD_DRV_LOG(ERR, "Unknown event was received or event with " 2690 "unimplemented handler\n"); 2691 } 2692 2693 static struct ena_aenq_handlers aenq_handlers = { 2694 .handlers = { 2695 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 2696 [ENA_ADMIN_NOTIFICATION] = ena_notification, 2697 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive 2698 }, 2699 .unimplemented_handler = unimplemented_aenq_handler 2700 }; 2701