1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev.h> 36 #include <rte_tcp.h> 37 #include <rte_atomic.h> 38 #include <rte_dev.h> 39 #include <rte_errno.h> 40 #include <rte_version.h> 41 #include <rte_eal_memconfig.h> 42 #include <rte_net.h> 43 44 #include "ena_ethdev.h" 45 #include "ena_logs.h" 46 #include "ena_platform.h" 47 #include "ena_com.h" 48 #include "ena_eth_com.h" 49 50 #include <ena_common_defs.h> 51 #include <ena_regs_defs.h> 52 #include <ena_admin_defs.h> 53 #include <ena_eth_io_defs.h> 54 55 #define DRV_MODULE_VER_MAJOR 1 56 #define DRV_MODULE_VER_MINOR 0 57 #define DRV_MODULE_VER_SUBMINOR 0 58 59 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 60 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 61 /*reverse version of ENA_IO_RXQ_IDX*/ 62 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 63 64 /* While processing submitted and completed descriptors (rx and tx path 65 * respectively) in a loop it is desired to: 66 * - perform batch submissions while populating sumbissmion queue 67 * - avoid blocking transmission of other packets during cleanup phase 68 * Hence the utilization ratio of 1/8 of a queue size. 69 */ 70 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 71 72 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 73 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 74 75 #define GET_L4_HDR_LEN(mbuf) \ 76 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 77 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 78 79 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 80 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 81 #define ENA_HASH_KEY_SIZE 40 82 #define ENA_ETH_SS_STATS 0xFF 83 #define ETH_GSTRING_LEN 32 84 85 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 86 87 enum ethtool_stringset { 88 ETH_SS_TEST = 0, 89 ETH_SS_STATS, 90 }; 91 92 struct ena_stats { 93 char name[ETH_GSTRING_LEN]; 94 int stat_offset; 95 }; 96 97 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 98 .name = #stat, \ 99 .stat_offset = offsetof(struct ena_com_stats_admin, stat) \ 100 } 101 102 #define ENA_STAT_ENTRY(stat, stat_type) { \ 103 .name = #stat, \ 104 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 105 } 106 107 #define ENA_STAT_RX_ENTRY(stat) \ 108 ENA_STAT_ENTRY(stat, rx) 109 110 #define ENA_STAT_TX_ENTRY(stat) \ 111 ENA_STAT_ENTRY(stat, tx) 112 113 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 114 ENA_STAT_ENTRY(stat, dev) 115 116 static const struct ena_stats ena_stats_global_strings[] = { 117 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 118 ENA_STAT_GLOBAL_ENTRY(io_suspend), 119 ENA_STAT_GLOBAL_ENTRY(io_resume), 120 ENA_STAT_GLOBAL_ENTRY(wd_expired), 121 ENA_STAT_GLOBAL_ENTRY(interface_up), 122 ENA_STAT_GLOBAL_ENTRY(interface_down), 123 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 124 }; 125 126 static const struct ena_stats ena_stats_tx_strings[] = { 127 ENA_STAT_TX_ENTRY(cnt), 128 ENA_STAT_TX_ENTRY(bytes), 129 ENA_STAT_TX_ENTRY(queue_stop), 130 ENA_STAT_TX_ENTRY(queue_wakeup), 131 ENA_STAT_TX_ENTRY(dma_mapping_err), 132 ENA_STAT_TX_ENTRY(linearize), 133 ENA_STAT_TX_ENTRY(linearize_failed), 134 ENA_STAT_TX_ENTRY(tx_poll), 135 ENA_STAT_TX_ENTRY(doorbells), 136 ENA_STAT_TX_ENTRY(prepare_ctx_err), 137 ENA_STAT_TX_ENTRY(missing_tx_comp), 138 ENA_STAT_TX_ENTRY(bad_req_id), 139 }; 140 141 static const struct ena_stats ena_stats_rx_strings[] = { 142 ENA_STAT_RX_ENTRY(cnt), 143 ENA_STAT_RX_ENTRY(bytes), 144 ENA_STAT_RX_ENTRY(refil_partial), 145 ENA_STAT_RX_ENTRY(bad_csum), 146 ENA_STAT_RX_ENTRY(page_alloc_fail), 147 ENA_STAT_RX_ENTRY(skb_alloc_fail), 148 ENA_STAT_RX_ENTRY(dma_mapping_err), 149 ENA_STAT_RX_ENTRY(bad_desc_num), 150 ENA_STAT_RX_ENTRY(small_copy_len_pkt), 151 }; 152 153 static const struct ena_stats ena_stats_ena_com_strings[] = { 154 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 155 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 156 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 157 ENA_STAT_ENA_COM_ENTRY(out_of_space), 158 ENA_STAT_ENA_COM_ENTRY(no_completion), 159 }; 160 161 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 162 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 163 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 164 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 165 166 /** Vendor ID used by Amazon devices */ 167 #define PCI_VENDOR_ID_AMAZON 0x1D0F 168 /** Amazon devices */ 169 #define PCI_DEVICE_ID_ENA_VF 0xEC20 170 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 171 172 #define ENA_TX_OFFLOAD_MASK (\ 173 PKT_TX_L4_MASK | \ 174 PKT_TX_IP_CKSUM | \ 175 PKT_TX_TCP_SEG) 176 177 #define ENA_TX_OFFLOAD_NOTSUP_MASK \ 178 (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) 179 180 static struct rte_pci_id pci_id_ena_map[] = { 181 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 182 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, 183 { .device_id = 0 }, 184 }; 185 186 static int ena_device_init(struct ena_com_dev *ena_dev, 187 struct ena_com_dev_get_features_ctx *get_feat_ctx); 188 static int ena_dev_configure(struct rte_eth_dev *dev); 189 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 190 uint16_t nb_pkts); 191 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 192 uint16_t nb_pkts); 193 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 194 uint16_t nb_desc, unsigned int socket_id, 195 const struct rte_eth_txconf *tx_conf); 196 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 197 uint16_t nb_desc, unsigned int socket_id, 198 const struct rte_eth_rxconf *rx_conf, 199 struct rte_mempool *mp); 200 static uint16_t eth_ena_recv_pkts(void *rx_queue, 201 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 202 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 203 static void ena_init_rings(struct ena_adapter *adapter); 204 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 205 static int ena_start(struct rte_eth_dev *dev); 206 static void ena_close(struct rte_eth_dev *dev); 207 static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 208 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 209 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 210 static void ena_rx_queue_release(void *queue); 211 static void ena_tx_queue_release(void *queue); 212 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 213 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 214 static int ena_link_update(struct rte_eth_dev *dev, 215 __rte_unused int wait_to_complete); 216 static int ena_queue_restart(struct ena_ring *ring); 217 static int ena_queue_restart_all(struct rte_eth_dev *dev, 218 enum ena_ring_type ring_type); 219 static void ena_stats_restart(struct rte_eth_dev *dev); 220 static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, 221 struct rte_eth_dev_info *dev_info); 222 static int ena_rss_reta_update(struct rte_eth_dev *dev, 223 struct rte_eth_rss_reta_entry64 *reta_conf, 224 uint16_t reta_size); 225 static int ena_rss_reta_query(struct rte_eth_dev *dev, 226 struct rte_eth_rss_reta_entry64 *reta_conf, 227 uint16_t reta_size); 228 static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); 229 230 static struct eth_dev_ops ena_dev_ops = { 231 .dev_configure = ena_dev_configure, 232 .dev_infos_get = ena_infos_get, 233 .rx_queue_setup = ena_rx_queue_setup, 234 .tx_queue_setup = ena_tx_queue_setup, 235 .dev_start = ena_start, 236 .link_update = ena_link_update, 237 .stats_get = ena_stats_get, 238 .mtu_set = ena_mtu_set, 239 .rx_queue_release = ena_rx_queue_release, 240 .tx_queue_release = ena_tx_queue_release, 241 .dev_close = ena_close, 242 .reta_update = ena_rss_reta_update, 243 .reta_query = ena_rss_reta_query, 244 }; 245 246 #define NUMA_NO_NODE SOCKET_ID_ANY 247 248 static inline int ena_cpu_to_node(int cpu) 249 { 250 struct rte_config *config = rte_eal_get_configuration(); 251 252 if (likely(cpu < RTE_MAX_MEMZONE)) 253 return config->mem_config->memzone[cpu].socket_id; 254 255 return NUMA_NO_NODE; 256 } 257 258 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 259 struct ena_com_rx_ctx *ena_rx_ctx) 260 { 261 uint64_t ol_flags = 0; 262 263 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 264 ol_flags |= PKT_TX_TCP_CKSUM; 265 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 266 ol_flags |= PKT_TX_UDP_CKSUM; 267 268 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 269 ol_flags |= PKT_TX_IPV4; 270 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 271 ol_flags |= PKT_TX_IPV6; 272 273 if (unlikely(ena_rx_ctx->l4_csum_err)) 274 ol_flags |= PKT_RX_L4_CKSUM_BAD; 275 if (unlikely(ena_rx_ctx->l3_csum_err)) 276 ol_flags |= PKT_RX_IP_CKSUM_BAD; 277 278 mbuf->ol_flags = ol_flags; 279 } 280 281 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 282 struct ena_com_tx_ctx *ena_tx_ctx) 283 { 284 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 285 286 if (mbuf->ol_flags & 287 (PKT_TX_L4_MASK | PKT_TX_IP_CKSUM | PKT_TX_TCP_SEG)) { 288 /* check if TSO is required */ 289 if (mbuf->ol_flags & PKT_TX_TCP_SEG) { 290 ena_tx_ctx->tso_enable = true; 291 292 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 293 } 294 295 /* check if L3 checksum is needed */ 296 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 297 ena_tx_ctx->l3_csum_enable = true; 298 299 if (mbuf->ol_flags & PKT_TX_IPV6) { 300 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 301 } else { 302 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 303 304 /* set don't fragment (DF) flag */ 305 if (mbuf->packet_type & 306 (RTE_PTYPE_L4_NONFRAG 307 | RTE_PTYPE_INNER_L4_NONFRAG)) 308 ena_tx_ctx->df = true; 309 } 310 311 /* check if L4 checksum is needed */ 312 switch (mbuf->ol_flags & PKT_TX_L4_MASK) { 313 case PKT_TX_TCP_CKSUM: 314 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 315 ena_tx_ctx->l4_csum_enable = true; 316 break; 317 case PKT_TX_UDP_CKSUM: 318 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 319 ena_tx_ctx->l4_csum_enable = true; 320 break; 321 default: 322 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 323 ena_tx_ctx->l4_csum_enable = false; 324 break; 325 } 326 327 ena_meta->mss = mbuf->tso_segsz; 328 ena_meta->l3_hdr_len = mbuf->l3_len; 329 ena_meta->l3_hdr_offset = mbuf->l2_len; 330 /* this param needed only for TSO */ 331 ena_meta->l3_outer_hdr_len = 0; 332 ena_meta->l3_outer_hdr_offset = 0; 333 334 ena_tx_ctx->meta_valid = true; 335 } else { 336 ena_tx_ctx->meta_valid = false; 337 } 338 } 339 340 static void ena_config_host_info(struct ena_com_dev *ena_dev) 341 { 342 struct ena_admin_host_info *host_info; 343 int rc; 344 345 /* Allocate only the host info */ 346 rc = ena_com_allocate_host_info(ena_dev); 347 if (rc) { 348 RTE_LOG(ERR, PMD, "Cannot allocate host info\n"); 349 return; 350 } 351 352 host_info = ena_dev->host_attr.host_info; 353 354 host_info->os_type = ENA_ADMIN_OS_DPDK; 355 host_info->kernel_ver = RTE_VERSION; 356 snprintf((char *)host_info->kernel_ver_str, 357 sizeof(host_info->kernel_ver_str), 358 "%s", rte_version()); 359 host_info->os_dist = RTE_VERSION; 360 snprintf((char *)host_info->os_dist_str, 361 sizeof(host_info->os_dist_str), 362 "%s", rte_version()); 363 host_info->driver_version = 364 (DRV_MODULE_VER_MAJOR) | 365 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 366 (DRV_MODULE_VER_SUBMINOR << 367 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 368 369 rc = ena_com_set_host_attributes(ena_dev); 370 if (rc) { 371 if (rc == -EPERM) 372 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 373 else 374 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 375 376 goto err; 377 } 378 379 return; 380 381 err: 382 ena_com_delete_host_info(ena_dev); 383 } 384 385 static int 386 ena_get_sset_count(struct rte_eth_dev *dev, int sset) 387 { 388 if (sset != ETH_SS_STATS) 389 return -EOPNOTSUPP; 390 391 /* Workaround for clang: 392 * touch internal structures to prevent 393 * compiler error 394 */ 395 ENA_TOUCH(ena_stats_global_strings); 396 ENA_TOUCH(ena_stats_tx_strings); 397 ENA_TOUCH(ena_stats_rx_strings); 398 ENA_TOUCH(ena_stats_ena_com_strings); 399 400 return dev->data->nb_tx_queues * 401 (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) + 402 ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 403 } 404 405 static void ena_config_debug_area(struct ena_adapter *adapter) 406 { 407 u32 debug_area_size; 408 int rc, ss_count; 409 410 ss_count = ena_get_sset_count(adapter->rte_dev, ETH_SS_STATS); 411 if (ss_count <= 0) { 412 RTE_LOG(ERR, PMD, "SS count is negative\n"); 413 return; 414 } 415 416 /* allocate 32 bytes for each string and 64bit for the value */ 417 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 418 419 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 420 if (rc) { 421 RTE_LOG(ERR, PMD, "Cannot allocate debug area\n"); 422 return; 423 } 424 425 rc = ena_com_set_host_attributes(&adapter->ena_dev); 426 if (rc) { 427 if (rc == -EPERM) 428 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 429 else 430 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 431 goto err; 432 } 433 434 return; 435 err: 436 ena_com_delete_debug_area(&adapter->ena_dev); 437 } 438 439 static void ena_close(struct rte_eth_dev *dev) 440 { 441 struct ena_adapter *adapter = 442 (struct ena_adapter *)(dev->data->dev_private); 443 444 adapter->state = ENA_ADAPTER_STATE_STOPPED; 445 446 ena_rx_queue_release_all(dev); 447 ena_tx_queue_release_all(dev); 448 } 449 450 static int ena_rss_reta_update(struct rte_eth_dev *dev, 451 struct rte_eth_rss_reta_entry64 *reta_conf, 452 uint16_t reta_size) 453 { 454 struct ena_adapter *adapter = 455 (struct ena_adapter *)(dev->data->dev_private); 456 struct ena_com_dev *ena_dev = &adapter->ena_dev; 457 int ret, i; 458 u16 entry_value; 459 int conf_idx; 460 int idx; 461 462 if ((reta_size == 0) || (reta_conf == NULL)) 463 return -EINVAL; 464 465 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 466 RTE_LOG(WARNING, PMD, 467 "indirection table %d is bigger than supported (%d)\n", 468 reta_size, ENA_RX_RSS_TABLE_SIZE); 469 ret = -EINVAL; 470 goto err; 471 } 472 473 for (i = 0 ; i < reta_size ; i++) { 474 /* each reta_conf is for 64 entries. 475 * to support 128 we use 2 conf of 64 476 */ 477 conf_idx = i / RTE_RETA_GROUP_SIZE; 478 idx = i % RTE_RETA_GROUP_SIZE; 479 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 480 entry_value = 481 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 482 ret = ena_com_indirect_table_fill_entry(ena_dev, 483 i, 484 entry_value); 485 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 486 RTE_LOG(ERR, PMD, 487 "Cannot fill indirect table\n"); 488 ret = -ENOTSUP; 489 goto err; 490 } 491 } 492 } 493 494 ret = ena_com_indirect_table_set(ena_dev); 495 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 496 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 497 ret = -ENOTSUP; 498 goto err; 499 } 500 501 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 502 __func__, reta_size, adapter->rte_dev->data->port_id); 503 err: 504 return ret; 505 } 506 507 /* Query redirection table. */ 508 static int ena_rss_reta_query(struct rte_eth_dev *dev, 509 struct rte_eth_rss_reta_entry64 *reta_conf, 510 uint16_t reta_size) 511 { 512 struct ena_adapter *adapter = 513 (struct ena_adapter *)(dev->data->dev_private); 514 struct ena_com_dev *ena_dev = &adapter->ena_dev; 515 int ret; 516 int i; 517 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 518 int reta_conf_idx; 519 int reta_idx; 520 521 if (reta_size == 0 || reta_conf == NULL || 522 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 523 return -EINVAL; 524 525 ret = ena_com_indirect_table_get(ena_dev, indirect_table); 526 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 527 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 528 ret = -ENOTSUP; 529 goto err; 530 } 531 532 for (i = 0 ; i < reta_size ; i++) { 533 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 534 reta_idx = i % RTE_RETA_GROUP_SIZE; 535 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 536 reta_conf[reta_conf_idx].reta[reta_idx] = 537 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 538 } 539 err: 540 return ret; 541 } 542 543 static int ena_rss_init_default(struct ena_adapter *adapter) 544 { 545 struct ena_com_dev *ena_dev = &adapter->ena_dev; 546 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 547 int rc, i; 548 u32 val; 549 550 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 551 if (unlikely(rc)) { 552 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 553 goto err_rss_init; 554 } 555 556 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 557 val = i % nb_rx_queues; 558 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 559 ENA_IO_RXQ_IDX(val)); 560 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 561 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 562 goto err_fill_indir; 563 } 564 } 565 566 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 567 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 568 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 569 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 570 goto err_fill_indir; 571 } 572 573 rc = ena_com_set_default_hash_ctrl(ena_dev); 574 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 575 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 576 goto err_fill_indir; 577 } 578 579 rc = ena_com_indirect_table_set(ena_dev); 580 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 581 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 582 goto err_fill_indir; 583 } 584 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 585 adapter->rte_dev->data->port_id); 586 587 return 0; 588 589 err_fill_indir: 590 ena_com_rss_destroy(ena_dev); 591 err_rss_init: 592 593 return rc; 594 } 595 596 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 597 { 598 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 599 int nb_queues = dev->data->nb_rx_queues; 600 int i; 601 602 for (i = 0; i < nb_queues; i++) 603 ena_rx_queue_release(queues[i]); 604 } 605 606 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 607 { 608 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 609 int nb_queues = dev->data->nb_tx_queues; 610 int i; 611 612 for (i = 0; i < nb_queues; i++) 613 ena_tx_queue_release(queues[i]); 614 } 615 616 static void ena_rx_queue_release(void *queue) 617 { 618 struct ena_ring *ring = (struct ena_ring *)queue; 619 struct ena_adapter *adapter = ring->adapter; 620 int ena_qid; 621 622 ena_assert_msg(ring->configured, 623 "API violation - releasing not configured queue"); 624 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 625 "API violation"); 626 627 /* Destroy HW queue */ 628 ena_qid = ENA_IO_RXQ_IDX(ring->id); 629 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 630 631 /* Free all bufs */ 632 ena_rx_queue_release_bufs(ring); 633 634 /* Free ring resources */ 635 if (ring->rx_buffer_info) 636 rte_free(ring->rx_buffer_info); 637 ring->rx_buffer_info = NULL; 638 639 ring->configured = 0; 640 641 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 642 ring->port_id, ring->id); 643 } 644 645 static void ena_tx_queue_release(void *queue) 646 { 647 struct ena_ring *ring = (struct ena_ring *)queue; 648 struct ena_adapter *adapter = ring->adapter; 649 int ena_qid; 650 651 ena_assert_msg(ring->configured, 652 "API violation. Releasing not configured queue"); 653 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 654 "API violation"); 655 656 /* Destroy HW queue */ 657 ena_qid = ENA_IO_TXQ_IDX(ring->id); 658 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 659 660 /* Free all bufs */ 661 ena_tx_queue_release_bufs(ring); 662 663 /* Free ring resources */ 664 if (ring->tx_buffer_info) 665 rte_free(ring->tx_buffer_info); 666 667 if (ring->empty_tx_reqs) 668 rte_free(ring->empty_tx_reqs); 669 670 ring->empty_tx_reqs = NULL; 671 ring->tx_buffer_info = NULL; 672 673 ring->configured = 0; 674 675 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 676 ring->port_id, ring->id); 677 } 678 679 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 680 { 681 unsigned int ring_mask = ring->ring_size - 1; 682 683 while (ring->next_to_clean != ring->next_to_use) { 684 struct rte_mbuf *m = 685 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 686 687 if (m) 688 __rte_mbuf_raw_free(m); 689 690 ring->next_to_clean++; 691 } 692 } 693 694 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 695 { 696 unsigned int ring_mask = ring->ring_size - 1; 697 698 while (ring->next_to_clean != ring->next_to_use) { 699 struct ena_tx_buffer *tx_buf = 700 &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; 701 702 if (tx_buf->mbuf) 703 rte_pktmbuf_free(tx_buf->mbuf); 704 705 ring->next_to_clean++; 706 } 707 } 708 709 static int ena_link_update(struct rte_eth_dev *dev, 710 __rte_unused int wait_to_complete) 711 { 712 struct rte_eth_link *link = &dev->data->dev_link; 713 714 link->link_status = 1; 715 link->link_speed = ETH_SPEED_NUM_10G; 716 link->link_duplex = ETH_LINK_FULL_DUPLEX; 717 718 return 0; 719 } 720 721 static int ena_queue_restart_all(struct rte_eth_dev *dev, 722 enum ena_ring_type ring_type) 723 { 724 struct ena_adapter *adapter = 725 (struct ena_adapter *)(dev->data->dev_private); 726 struct ena_ring *queues = NULL; 727 int i = 0; 728 int rc = 0; 729 730 queues = (ring_type == ENA_RING_TYPE_RX) ? 731 adapter->rx_ring : adapter->tx_ring; 732 733 for (i = 0; i < adapter->num_queues; i++) { 734 if (queues[i].configured) { 735 if (ring_type == ENA_RING_TYPE_RX) { 736 ena_assert_msg( 737 dev->data->rx_queues[i] == &queues[i], 738 "Inconsistent state of rx queues\n"); 739 } else { 740 ena_assert_msg( 741 dev->data->tx_queues[i] == &queues[i], 742 "Inconsistent state of tx queues\n"); 743 } 744 745 rc = ena_queue_restart(&queues[i]); 746 747 if (rc) { 748 PMD_INIT_LOG(ERR, 749 "failed to restart queue %d type(%d)\n", 750 i, ring_type); 751 return -1; 752 } 753 } 754 } 755 756 return 0; 757 } 758 759 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 760 { 761 uint32_t max_frame_len = adapter->max_mtu; 762 763 if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) 764 max_frame_len = 765 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 766 767 return max_frame_len; 768 } 769 770 static int ena_check_valid_conf(struct ena_adapter *adapter) 771 { 772 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 773 774 if (max_frame_len > adapter->max_mtu) { 775 PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n", max_frame_len); 776 return -1; 777 } 778 779 return 0; 780 } 781 782 static int 783 ena_calc_queue_size(struct ena_com_dev *ena_dev, 784 struct ena_com_dev_get_features_ctx *get_feat_ctx) 785 { 786 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 787 788 queue_size = RTE_MIN(queue_size, 789 get_feat_ctx->max_queues.max_cq_depth); 790 queue_size = RTE_MIN(queue_size, 791 get_feat_ctx->max_queues.max_sq_depth); 792 793 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 794 queue_size = RTE_MIN(queue_size, 795 get_feat_ctx->max_queues.max_llq_depth); 796 797 /* Round down to power of 2 */ 798 if (!rte_is_power_of_2(queue_size)) 799 queue_size = rte_align32pow2(queue_size >> 1); 800 801 if (queue_size == 0) { 802 PMD_INIT_LOG(ERR, "Invalid queue size\n"); 803 return -EFAULT; 804 } 805 806 return queue_size; 807 } 808 809 static void ena_stats_restart(struct rte_eth_dev *dev) 810 { 811 struct ena_adapter *adapter = 812 (struct ena_adapter *)(dev->data->dev_private); 813 814 rte_atomic64_init(&adapter->drv_stats->ierrors); 815 rte_atomic64_init(&adapter->drv_stats->oerrors); 816 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 817 } 818 819 static void ena_stats_get(struct rte_eth_dev *dev, 820 struct rte_eth_stats *stats) 821 { 822 struct ena_admin_basic_stats ena_stats; 823 struct ena_adapter *adapter = 824 (struct ena_adapter *)(dev->data->dev_private); 825 struct ena_com_dev *ena_dev = &adapter->ena_dev; 826 int rc; 827 828 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 829 return; 830 831 memset(&ena_stats, 0, sizeof(ena_stats)); 832 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 833 if (unlikely(rc)) { 834 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 835 return; 836 } 837 838 /* Set of basic statistics from ENA */ 839 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 840 ena_stats.rx_pkts_low); 841 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 842 ena_stats.tx_pkts_low); 843 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 844 ena_stats.rx_bytes_low); 845 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 846 ena_stats.tx_bytes_low); 847 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 848 ena_stats.rx_drops_low); 849 850 /* Driver related stats */ 851 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 852 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 853 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 854 } 855 856 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 857 { 858 struct ena_adapter *adapter; 859 struct ena_com_dev *ena_dev; 860 int rc = 0; 861 862 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 863 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 864 adapter = (struct ena_adapter *)(dev->data->dev_private); 865 866 ena_dev = &adapter->ena_dev; 867 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 868 869 if (mtu > ena_get_mtu_conf(adapter)) { 870 RTE_LOG(ERR, PMD, 871 "Given MTU (%d) exceeds maximum MTU supported (%d)\n", 872 mtu, ena_get_mtu_conf(adapter)); 873 rc = -EINVAL; 874 goto err; 875 } 876 877 rc = ena_com_set_dev_mtu(ena_dev, mtu); 878 if (rc) 879 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 880 else 881 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 882 883 err: 884 return rc; 885 } 886 887 static int ena_start(struct rte_eth_dev *dev) 888 { 889 struct ena_adapter *adapter = 890 (struct ena_adapter *)(dev->data->dev_private); 891 int rc = 0; 892 893 if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || 894 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 895 PMD_INIT_LOG(ERR, "API violation"); 896 return -1; 897 } 898 899 rc = ena_check_valid_conf(adapter); 900 if (rc) 901 return rc; 902 903 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 904 if (rc) 905 return rc; 906 907 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 908 if (rc) 909 return rc; 910 911 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 912 ETH_MQ_RX_RSS_FLAG) { 913 rc = ena_rss_init_default(adapter); 914 if (rc) 915 return rc; 916 } 917 918 ena_stats_restart(dev); 919 920 adapter->state = ENA_ADAPTER_STATE_RUNNING; 921 922 return 0; 923 } 924 925 static int ena_queue_restart(struct ena_ring *ring) 926 { 927 int rc; 928 929 ena_assert_msg(ring->configured == 1, 930 "Trying to restart unconfigured queue\n"); 931 932 ring->next_to_clean = 0; 933 ring->next_to_use = 0; 934 935 if (ring->type == ENA_RING_TYPE_TX) 936 return 0; 937 938 rc = ena_populate_rx_queue(ring, ring->ring_size); 939 if ((unsigned int)rc != ring->ring_size) { 940 PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n"); 941 return (-1); 942 } 943 944 return 0; 945 } 946 947 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 948 uint16_t queue_idx, 949 uint16_t nb_desc, 950 __rte_unused unsigned int socket_id, 951 __rte_unused const struct rte_eth_txconf *tx_conf) 952 { 953 struct ena_com_create_io_ctx ctx = 954 /* policy set to _HOST just to satisfy icc compiler */ 955 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 956 ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 }; 957 struct ena_ring *txq = NULL; 958 struct ena_adapter *adapter = 959 (struct ena_adapter *)(dev->data->dev_private); 960 unsigned int i; 961 int ena_qid; 962 int rc; 963 struct ena_com_dev *ena_dev = &adapter->ena_dev; 964 965 txq = &adapter->tx_ring[queue_idx]; 966 967 if (txq->configured) { 968 RTE_LOG(CRIT, PMD, 969 "API violation. Queue %d is already configured\n", 970 queue_idx); 971 return -1; 972 } 973 974 if (!rte_is_power_of_2(nb_desc)) { 975 RTE_LOG(ERR, PMD, 976 "Unsupported size of RX queue: %d is not a power of 2.", 977 nb_desc); 978 return -EINVAL; 979 } 980 981 if (nb_desc > adapter->tx_ring_size) { 982 RTE_LOG(ERR, PMD, 983 "Unsupported size of TX queue (max size: %d)\n", 984 adapter->tx_ring_size); 985 return -EINVAL; 986 } 987 988 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 989 990 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 991 ctx.qid = ena_qid; 992 ctx.msix_vector = -1; /* admin interrupts not used */ 993 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 994 ctx.queue_size = adapter->tx_ring_size; 995 ctx.numa_node = ena_cpu_to_node(queue_idx); 996 997 rc = ena_com_create_io_queue(ena_dev, &ctx); 998 if (rc) { 999 RTE_LOG(ERR, PMD, 1000 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 1001 queue_idx, ena_qid, rc); 1002 } 1003 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 1004 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1005 1006 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1007 &txq->ena_com_io_sq, 1008 &txq->ena_com_io_cq); 1009 if (rc) { 1010 RTE_LOG(ERR, PMD, 1011 "Failed to get TX queue handlers. TX queue num %d rc: %d\n", 1012 queue_idx, rc); 1013 ena_com_destroy_io_queue(ena_dev, ena_qid); 1014 goto err; 1015 } 1016 1017 txq->port_id = dev->data->port_id; 1018 txq->next_to_clean = 0; 1019 txq->next_to_use = 0; 1020 txq->ring_size = nb_desc; 1021 1022 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1023 sizeof(struct ena_tx_buffer) * 1024 txq->ring_size, 1025 RTE_CACHE_LINE_SIZE); 1026 if (!txq->tx_buffer_info) { 1027 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 1028 return -ENOMEM; 1029 } 1030 1031 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1032 sizeof(u16) * txq->ring_size, 1033 RTE_CACHE_LINE_SIZE); 1034 if (!txq->empty_tx_reqs) { 1035 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 1036 rte_free(txq->tx_buffer_info); 1037 return -ENOMEM; 1038 } 1039 for (i = 0; i < txq->ring_size; i++) 1040 txq->empty_tx_reqs[i] = i; 1041 1042 /* Store pointer to this queue in upper layer */ 1043 txq->configured = 1; 1044 dev->data->tx_queues[queue_idx] = txq; 1045 err: 1046 return rc; 1047 } 1048 1049 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1050 uint16_t queue_idx, 1051 uint16_t nb_desc, 1052 __rte_unused unsigned int socket_id, 1053 __rte_unused const struct rte_eth_rxconf *rx_conf, 1054 struct rte_mempool *mp) 1055 { 1056 struct ena_com_create_io_ctx ctx = 1057 /* policy set to _HOST just to satisfy icc compiler */ 1058 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1059 ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 }; 1060 struct ena_adapter *adapter = 1061 (struct ena_adapter *)(dev->data->dev_private); 1062 struct ena_ring *rxq = NULL; 1063 uint16_t ena_qid = 0; 1064 int rc = 0; 1065 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1066 1067 rxq = &adapter->rx_ring[queue_idx]; 1068 if (rxq->configured) { 1069 RTE_LOG(CRIT, PMD, 1070 "API violation. Queue %d is already configured\n", 1071 queue_idx); 1072 return -1; 1073 } 1074 1075 if (!rte_is_power_of_2(nb_desc)) { 1076 RTE_LOG(ERR, PMD, 1077 "Unsupported size of TX queue: %d is not a power of 2.", 1078 nb_desc); 1079 return -EINVAL; 1080 } 1081 1082 if (nb_desc > adapter->rx_ring_size) { 1083 RTE_LOG(ERR, PMD, 1084 "Unsupported size of RX queue (max size: %d)\n", 1085 adapter->rx_ring_size); 1086 return -EINVAL; 1087 } 1088 1089 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 1090 1091 ctx.qid = ena_qid; 1092 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1093 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1094 ctx.msix_vector = -1; /* admin interrupts not used */ 1095 ctx.queue_size = adapter->rx_ring_size; 1096 ctx.numa_node = ena_cpu_to_node(queue_idx); 1097 1098 rc = ena_com_create_io_queue(ena_dev, &ctx); 1099 if (rc) 1100 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 1101 queue_idx, rc); 1102 1103 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 1104 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1105 1106 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1107 &rxq->ena_com_io_sq, 1108 &rxq->ena_com_io_cq); 1109 if (rc) { 1110 RTE_LOG(ERR, PMD, 1111 "Failed to get RX queue handlers. RX queue num %d rc: %d\n", 1112 queue_idx, rc); 1113 ena_com_destroy_io_queue(ena_dev, ena_qid); 1114 } 1115 1116 rxq->port_id = dev->data->port_id; 1117 rxq->next_to_clean = 0; 1118 rxq->next_to_use = 0; 1119 rxq->ring_size = nb_desc; 1120 rxq->mb_pool = mp; 1121 1122 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1123 sizeof(struct rte_mbuf *) * nb_desc, 1124 RTE_CACHE_LINE_SIZE); 1125 if (!rxq->rx_buffer_info) { 1126 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 1127 return -ENOMEM; 1128 } 1129 1130 /* Store pointer to this queue in upper layer */ 1131 rxq->configured = 1; 1132 dev->data->rx_queues[queue_idx] = rxq; 1133 1134 return rc; 1135 } 1136 1137 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1138 { 1139 unsigned int i; 1140 int rc; 1141 uint16_t ring_size = rxq->ring_size; 1142 uint16_t ring_mask = ring_size - 1; 1143 uint16_t next_to_use = rxq->next_to_use; 1144 uint16_t in_use; 1145 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 1146 1147 if (unlikely(!count)) 1148 return 0; 1149 1150 in_use = rxq->next_to_use - rxq->next_to_clean; 1151 ena_assert_msg(((in_use + count) <= ring_size), "bad ring state"); 1152 1153 count = RTE_MIN(count, 1154 (uint16_t)(ring_size - (next_to_use & ring_mask))); 1155 1156 /* get resources for incoming packets */ 1157 rc = rte_mempool_get_bulk(rxq->mb_pool, 1158 (void **)(&mbufs[next_to_use & ring_mask]), 1159 count); 1160 if (unlikely(rc < 0)) { 1161 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1162 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1163 return 0; 1164 } 1165 1166 for (i = 0; i < count; i++) { 1167 uint16_t next_to_use_masked = next_to_use & ring_mask; 1168 struct rte_mbuf *mbuf = mbufs[next_to_use_masked]; 1169 struct ena_com_buf ebuf; 1170 1171 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 1172 /* prepare physical address for DMA transaction */ 1173 ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; 1174 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1175 /* pass resource to device */ 1176 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 1177 &ebuf, next_to_use_masked); 1178 if (unlikely(rc)) { 1179 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 1180 break; 1181 } 1182 next_to_use++; 1183 } 1184 1185 /* When we submitted free recources to device... */ 1186 if (i > 0) { 1187 /* ...let HW know that it can fill buffers with data */ 1188 rte_wmb(); 1189 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1190 1191 rxq->next_to_use = next_to_use; 1192 } 1193 1194 return i; 1195 } 1196 1197 static int ena_device_init(struct ena_com_dev *ena_dev, 1198 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1199 { 1200 int rc; 1201 bool readless_supported; 1202 1203 /* Initialize mmio registers */ 1204 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1205 if (rc) { 1206 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 1207 return rc; 1208 } 1209 1210 /* The PCIe configuration space revision id indicate if mmio reg 1211 * read is disabled. 1212 */ 1213 readless_supported = 1214 !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id 1215 & ENA_MMIO_DISABLE_REG_READ); 1216 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1217 1218 /* reset device */ 1219 rc = ena_com_dev_reset(ena_dev); 1220 if (rc) { 1221 RTE_LOG(ERR, PMD, "cannot reset device\n"); 1222 goto err_mmio_read_less; 1223 } 1224 1225 /* check FW version */ 1226 rc = ena_com_validate_version(ena_dev); 1227 if (rc) { 1228 RTE_LOG(ERR, PMD, "device version is too low\n"); 1229 goto err_mmio_read_less; 1230 } 1231 1232 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1233 1234 /* ENA device administration layer init */ 1235 rc = ena_com_admin_init(ena_dev, NULL, true); 1236 if (rc) { 1237 RTE_LOG(ERR, PMD, 1238 "cannot initialize ena admin queue with device\n"); 1239 goto err_mmio_read_less; 1240 } 1241 1242 ena_config_host_info(ena_dev); 1243 1244 /* To enable the msix interrupts the driver needs to know the number 1245 * of queues. So the driver uses polling mode to retrieve this 1246 * information. 1247 */ 1248 ena_com_set_admin_polling_mode(ena_dev, true); 1249 1250 /* Get Device Attributes and features */ 1251 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1252 if (rc) { 1253 RTE_LOG(ERR, PMD, 1254 "cannot get attribute for ena device rc= %d\n", rc); 1255 goto err_admin_init; 1256 } 1257 1258 return 0; 1259 1260 err_admin_init: 1261 ena_com_admin_destroy(ena_dev); 1262 1263 err_mmio_read_less: 1264 ena_com_mmio_reg_read_request_destroy(ena_dev); 1265 1266 return rc; 1267 } 1268 1269 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1270 { 1271 struct rte_pci_device *pci_dev; 1272 struct ena_adapter *adapter = 1273 (struct ena_adapter *)(eth_dev->data->dev_private); 1274 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1275 struct ena_com_dev_get_features_ctx get_feat_ctx; 1276 int queue_size, rc; 1277 1278 static int adapters_found; 1279 1280 memset(adapter, 0, sizeof(struct ena_adapter)); 1281 ena_dev = &adapter->ena_dev; 1282 1283 eth_dev->dev_ops = &ena_dev_ops; 1284 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1285 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1286 eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; 1287 adapter->rte_eth_dev_data = eth_dev->data; 1288 adapter->rte_dev = eth_dev; 1289 1290 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1291 return 0; 1292 1293 pci_dev = RTE_DEV_TO_PCI(eth_dev->device); 1294 adapter->pdev = pci_dev; 1295 1296 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", 1297 pci_dev->addr.domain, 1298 pci_dev->addr.bus, 1299 pci_dev->addr.devid, 1300 pci_dev->addr.function); 1301 1302 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1303 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1304 1305 /* Present ENA_MEM_BAR indicates available LLQ mode. 1306 * Use corresponding policy 1307 */ 1308 if (adapter->dev_mem_base) 1309 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 1310 else if (adapter->regs) 1311 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1312 else 1313 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", 1314 ENA_REGS_BAR); 1315 1316 ena_dev->reg_bar = adapter->regs; 1317 ena_dev->dmadev = adapter->pdev; 1318 1319 adapter->id_number = adapters_found; 1320 1321 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1322 adapter->id_number); 1323 1324 /* device specific initialization routine */ 1325 rc = ena_device_init(ena_dev, &get_feat_ctx); 1326 if (rc) { 1327 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); 1328 return -1; 1329 } 1330 1331 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1332 if (get_feat_ctx.max_queues.max_llq_num == 0) { 1333 PMD_INIT_LOG(ERR, 1334 "Trying to use LLQ but llq_num is 0.\n" 1335 "Fall back into regular queues.\n"); 1336 ena_dev->tx_mem_queue_type = 1337 ENA_ADMIN_PLACEMENT_POLICY_HOST; 1338 adapter->num_queues = 1339 get_feat_ctx.max_queues.max_sq_num; 1340 } else { 1341 adapter->num_queues = 1342 get_feat_ctx.max_queues.max_llq_num; 1343 } 1344 } else { 1345 adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; 1346 } 1347 1348 queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); 1349 if ((queue_size <= 0) || (adapter->num_queues <= 0)) 1350 return -EFAULT; 1351 1352 adapter->tx_ring_size = queue_size; 1353 adapter->rx_ring_size = queue_size; 1354 1355 /* prepare ring structures */ 1356 ena_init_rings(adapter); 1357 1358 ena_config_debug_area(adapter); 1359 1360 /* Set max MTU for this device */ 1361 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1362 1363 /* Copy MAC address and point DPDK to it */ 1364 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1365 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1366 (struct ether_addr *)adapter->mac_addr); 1367 1368 adapter->drv_stats = rte_zmalloc("adapter stats", 1369 sizeof(*adapter->drv_stats), 1370 RTE_CACHE_LINE_SIZE); 1371 if (!adapter->drv_stats) { 1372 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1373 return -ENOMEM; 1374 } 1375 1376 adapters_found++; 1377 adapter->state = ENA_ADAPTER_STATE_INIT; 1378 1379 return 0; 1380 } 1381 1382 static int ena_dev_configure(struct rte_eth_dev *dev) 1383 { 1384 struct ena_adapter *adapter = 1385 (struct ena_adapter *)(dev->data->dev_private); 1386 1387 if (!(adapter->state == ENA_ADAPTER_STATE_INIT || 1388 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 1389 PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n", 1390 adapter->state); 1391 return -1; 1392 } 1393 1394 switch (adapter->state) { 1395 case ENA_ADAPTER_STATE_INIT: 1396 case ENA_ADAPTER_STATE_STOPPED: 1397 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1398 break; 1399 case ENA_ADAPTER_STATE_CONFIG: 1400 RTE_LOG(WARNING, PMD, 1401 "Ivalid driver state while trying to configure device\n"); 1402 break; 1403 default: 1404 break; 1405 } 1406 1407 return 0; 1408 } 1409 1410 static void ena_init_rings(struct ena_adapter *adapter) 1411 { 1412 int i; 1413 1414 for (i = 0; i < adapter->num_queues; i++) { 1415 struct ena_ring *ring = &adapter->tx_ring[i]; 1416 1417 ring->configured = 0; 1418 ring->type = ENA_RING_TYPE_TX; 1419 ring->adapter = adapter; 1420 ring->id = i; 1421 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1422 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1423 } 1424 1425 for (i = 0; i < adapter->num_queues; i++) { 1426 struct ena_ring *ring = &adapter->rx_ring[i]; 1427 1428 ring->configured = 0; 1429 ring->type = ENA_RING_TYPE_RX; 1430 ring->adapter = adapter; 1431 ring->id = i; 1432 } 1433 } 1434 1435 static void ena_infos_get(struct rte_eth_dev *dev, 1436 struct rte_eth_dev_info *dev_info) 1437 { 1438 struct ena_adapter *adapter; 1439 struct ena_com_dev *ena_dev; 1440 struct ena_com_dev_get_features_ctx feat; 1441 uint32_t rx_feat = 0, tx_feat = 0; 1442 int rc = 0; 1443 1444 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1445 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1446 adapter = (struct ena_adapter *)(dev->data->dev_private); 1447 1448 ena_dev = &adapter->ena_dev; 1449 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1450 1451 dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); 1452 1453 dev_info->speed_capa = 1454 ETH_LINK_SPEED_1G | 1455 ETH_LINK_SPEED_2_5G | 1456 ETH_LINK_SPEED_5G | 1457 ETH_LINK_SPEED_10G | 1458 ETH_LINK_SPEED_25G | 1459 ETH_LINK_SPEED_40G | 1460 ETH_LINK_SPEED_50G | 1461 ETH_LINK_SPEED_100G; 1462 1463 /* Get supported features from HW */ 1464 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1465 if (unlikely(rc)) { 1466 RTE_LOG(ERR, PMD, 1467 "Cannot get attribute for ena device rc= %d\n", rc); 1468 return; 1469 } 1470 1471 /* Set Tx & Rx features available for device */ 1472 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1473 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1474 1475 if (feat.offload.tx & 1476 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1477 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1478 DEV_TX_OFFLOAD_UDP_CKSUM | 1479 DEV_TX_OFFLOAD_TCP_CKSUM; 1480 1481 if (feat.offload.tx & 1482 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1483 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1484 DEV_RX_OFFLOAD_UDP_CKSUM | 1485 DEV_RX_OFFLOAD_TCP_CKSUM; 1486 1487 /* Inform framework about available features */ 1488 dev_info->rx_offload_capa = rx_feat; 1489 dev_info->tx_offload_capa = tx_feat; 1490 1491 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1492 dev_info->max_rx_pktlen = adapter->max_mtu; 1493 dev_info->max_mac_addrs = 1; 1494 1495 dev_info->max_rx_queues = adapter->num_queues; 1496 dev_info->max_tx_queues = adapter->num_queues; 1497 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1498 } 1499 1500 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1501 uint16_t nb_pkts) 1502 { 1503 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1504 unsigned int ring_size = rx_ring->ring_size; 1505 unsigned int ring_mask = ring_size - 1; 1506 uint16_t next_to_clean = rx_ring->next_to_clean; 1507 uint16_t desc_in_use = 0; 1508 unsigned int recv_idx = 0; 1509 struct rte_mbuf *mbuf = NULL; 1510 struct rte_mbuf *mbuf_head = NULL; 1511 struct rte_mbuf *mbuf_prev = NULL; 1512 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1513 unsigned int completed; 1514 1515 struct ena_com_rx_ctx ena_rx_ctx; 1516 int rc = 0; 1517 1518 /* Check adapter state */ 1519 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1520 RTE_LOG(ALERT, PMD, 1521 "Trying to receive pkts while device is NOT running\n"); 1522 return 0; 1523 } 1524 1525 desc_in_use = rx_ring->next_to_use - next_to_clean; 1526 if (unlikely(nb_pkts > desc_in_use)) 1527 nb_pkts = desc_in_use; 1528 1529 for (completed = 0; completed < nb_pkts; completed++) { 1530 int segments = 0; 1531 1532 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1533 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1534 ena_rx_ctx.descs = 0; 1535 /* receive packet context */ 1536 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1537 rx_ring->ena_com_io_sq, 1538 &ena_rx_ctx); 1539 if (unlikely(rc)) { 1540 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1541 return 0; 1542 } 1543 1544 if (unlikely(ena_rx_ctx.descs == 0)) 1545 break; 1546 1547 while (segments < ena_rx_ctx.descs) { 1548 mbuf = rx_buff_info[next_to_clean & ring_mask]; 1549 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1550 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1551 mbuf->refcnt = 1; 1552 mbuf->next = NULL; 1553 if (segments == 0) { 1554 mbuf->nb_segs = ena_rx_ctx.descs; 1555 mbuf->port = rx_ring->port_id; 1556 mbuf->pkt_len = 0; 1557 mbuf_head = mbuf; 1558 } else { 1559 /* for multi-segment pkts create mbuf chain */ 1560 mbuf_prev->next = mbuf; 1561 } 1562 mbuf_head->pkt_len += mbuf->data_len; 1563 1564 mbuf_prev = mbuf; 1565 segments++; 1566 next_to_clean++; 1567 } 1568 1569 /* fill mbuf attributes if any */ 1570 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1571 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1572 1573 /* pass to DPDK application head mbuf */ 1574 rx_pkts[recv_idx] = mbuf_head; 1575 recv_idx++; 1576 } 1577 1578 /* Burst refill to save doorbells, memory barriers, const interval */ 1579 if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) 1580 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use); 1581 1582 rx_ring->next_to_clean = next_to_clean; 1583 1584 return recv_idx; 1585 } 1586 1587 static uint16_t 1588 eth_ena_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 1589 uint16_t nb_pkts) 1590 { 1591 int32_t ret; 1592 uint32_t i; 1593 struct rte_mbuf *m; 1594 uint64_t ol_flags; 1595 1596 for (i = 0; i != nb_pkts; i++) { 1597 m = tx_pkts[i]; 1598 ol_flags = m->ol_flags; 1599 1600 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || 1601 (ol_flags & PKT_TX_L4_MASK) == 1602 PKT_TX_SCTP_CKSUM) { 1603 rte_errno = -ENOTSUP; 1604 return i; 1605 } 1606 1607 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1608 ret = rte_validate_tx_offload(m); 1609 if (ret != 0) { 1610 rte_errno = ret; 1611 return i; 1612 } 1613 #endif 1614 /* ENA doesn't need different phdr cskum for TSO */ 1615 ret = rte_net_intel_cksum_flags_prepare(m, 1616 ol_flags & ~PKT_TX_TCP_SEG); 1617 if (ret != 0) { 1618 rte_errno = ret; 1619 return i; 1620 } 1621 } 1622 1623 return i; 1624 } 1625 1626 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1627 uint16_t nb_pkts) 1628 { 1629 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1630 uint16_t next_to_use = tx_ring->next_to_use; 1631 uint16_t next_to_clean = tx_ring->next_to_clean; 1632 struct rte_mbuf *mbuf; 1633 unsigned int ring_size = tx_ring->ring_size; 1634 unsigned int ring_mask = ring_size - 1; 1635 struct ena_com_tx_ctx ena_tx_ctx; 1636 struct ena_tx_buffer *tx_info; 1637 struct ena_com_buf *ebuf; 1638 uint16_t rc, req_id, total_tx_descs = 0; 1639 uint16_t sent_idx = 0, empty_tx_reqs; 1640 int nb_hw_desc; 1641 1642 /* Check adapter state */ 1643 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1644 RTE_LOG(ALERT, PMD, 1645 "Trying to xmit pkts while device is NOT running\n"); 1646 return 0; 1647 } 1648 1649 empty_tx_reqs = ring_size - (next_to_use - next_to_clean); 1650 if (nb_pkts > empty_tx_reqs) 1651 nb_pkts = empty_tx_reqs; 1652 1653 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 1654 mbuf = tx_pkts[sent_idx]; 1655 1656 req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; 1657 tx_info = &tx_ring->tx_buffer_info[req_id]; 1658 tx_info->mbuf = mbuf; 1659 tx_info->num_of_bufs = 0; 1660 ebuf = tx_info->bufs; 1661 1662 /* Prepare TX context */ 1663 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1664 memset(&ena_tx_ctx.ena_meta, 0x0, 1665 sizeof(struct ena_com_tx_meta)); 1666 ena_tx_ctx.ena_bufs = ebuf; 1667 ena_tx_ctx.req_id = req_id; 1668 if (tx_ring->tx_mem_queue_type == 1669 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1670 /* prepare the push buffer with 1671 * virtual address of the data 1672 */ 1673 ena_tx_ctx.header_len = 1674 RTE_MIN(mbuf->data_len, 1675 tx_ring->tx_max_header_size); 1676 ena_tx_ctx.push_header = 1677 (void *)((char *)mbuf->buf_addr + 1678 mbuf->data_off); 1679 } /* there's no else as we take advantage of memset zeroing */ 1680 1681 /* Set TX offloads flags, if applicable */ 1682 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); 1683 1684 if (unlikely(mbuf->ol_flags & 1685 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 1686 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 1687 1688 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 1689 1690 /* Process first segment taking into 1691 * consideration pushed header 1692 */ 1693 if (mbuf->data_len > ena_tx_ctx.header_len) { 1694 ebuf->paddr = mbuf->buf_physaddr + 1695 mbuf->data_off + 1696 ena_tx_ctx.header_len; 1697 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 1698 ebuf++; 1699 tx_info->num_of_bufs++; 1700 } 1701 1702 while ((mbuf = mbuf->next) != NULL) { 1703 ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; 1704 ebuf->len = mbuf->data_len; 1705 ebuf++; 1706 tx_info->num_of_bufs++; 1707 } 1708 1709 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1710 1711 /* Write data to device */ 1712 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 1713 &ena_tx_ctx, &nb_hw_desc); 1714 if (unlikely(rc)) 1715 break; 1716 1717 tx_info->tx_descs = nb_hw_desc; 1718 1719 next_to_use++; 1720 } 1721 1722 /* If there are ready packets to be xmitted... */ 1723 if (sent_idx > 0) { 1724 /* ...let HW do its best :-) */ 1725 rte_wmb(); 1726 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 1727 1728 tx_ring->next_to_use = next_to_use; 1729 } 1730 1731 /* Clear complete packets */ 1732 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 1733 /* Get Tx info & store how many descs were processed */ 1734 tx_info = &tx_ring->tx_buffer_info[req_id]; 1735 total_tx_descs += tx_info->tx_descs; 1736 1737 /* Free whole mbuf chain */ 1738 mbuf = tx_info->mbuf; 1739 rte_pktmbuf_free(mbuf); 1740 1741 /* Put back descriptor to the ring for reuse */ 1742 tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; 1743 next_to_clean++; 1744 1745 /* If too many descs to clean, leave it for another run */ 1746 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 1747 break; 1748 } 1749 1750 if (total_tx_descs > 0) { 1751 /* acknowledge completion of sent packets */ 1752 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 1753 tx_ring->next_to_clean = next_to_clean; 1754 } 1755 1756 return sent_idx; 1757 } 1758 1759 static struct eth_driver rte_ena_pmd = { 1760 .pci_drv = { 1761 .id_table = pci_id_ena_map, 1762 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1763 .probe = rte_eth_dev_pci_probe, 1764 .remove = rte_eth_dev_pci_remove, 1765 }, 1766 .eth_dev_init = eth_ena_dev_init, 1767 .dev_private_size = sizeof(struct ena_adapter), 1768 }; 1769 1770 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd.pci_drv); 1771 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 1772 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio"); 1773