1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev.h> 36 #include <rte_tcp.h> 37 #include <rte_atomic.h> 38 #include <rte_dev.h> 39 #include <rte_errno.h> 40 #include <rte_version.h> 41 #include <rte_eal_memconfig.h> 42 #include <rte_net.h> 43 44 #include "ena_ethdev.h" 45 #include "ena_logs.h" 46 #include "ena_platform.h" 47 #include "ena_com.h" 48 #include "ena_eth_com.h" 49 50 #include <ena_common_defs.h> 51 #include <ena_regs_defs.h> 52 #include <ena_admin_defs.h> 53 #include <ena_eth_io_defs.h> 54 55 #define DRV_MODULE_VER_MAJOR 1 56 #define DRV_MODULE_VER_MINOR 0 57 #define DRV_MODULE_VER_SUBMINOR 0 58 59 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 60 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 61 /*reverse version of ENA_IO_RXQ_IDX*/ 62 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 63 64 /* While processing submitted and completed descriptors (rx and tx path 65 * respectively) in a loop it is desired to: 66 * - perform batch submissions while populating sumbissmion queue 67 * - avoid blocking transmission of other packets during cleanup phase 68 * Hence the utilization ratio of 1/8 of a queue size. 69 */ 70 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 71 72 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 73 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 74 75 #define GET_L4_HDR_LEN(mbuf) \ 76 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 77 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 78 79 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 80 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 81 #define ENA_HASH_KEY_SIZE 40 82 #define ENA_ETH_SS_STATS 0xFF 83 #define ETH_GSTRING_LEN 32 84 85 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 86 87 enum ethtool_stringset { 88 ETH_SS_TEST = 0, 89 ETH_SS_STATS, 90 }; 91 92 struct ena_stats { 93 char name[ETH_GSTRING_LEN]; 94 int stat_offset; 95 }; 96 97 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 98 .name = #stat, \ 99 .stat_offset = offsetof(struct ena_com_stats_admin, stat) \ 100 } 101 102 #define ENA_STAT_ENTRY(stat, stat_type) { \ 103 .name = #stat, \ 104 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 105 } 106 107 #define ENA_STAT_RX_ENTRY(stat) \ 108 ENA_STAT_ENTRY(stat, rx) 109 110 #define ENA_STAT_TX_ENTRY(stat) \ 111 ENA_STAT_ENTRY(stat, tx) 112 113 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 114 ENA_STAT_ENTRY(stat, dev) 115 116 static const struct ena_stats ena_stats_global_strings[] = { 117 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 118 ENA_STAT_GLOBAL_ENTRY(io_suspend), 119 ENA_STAT_GLOBAL_ENTRY(io_resume), 120 ENA_STAT_GLOBAL_ENTRY(wd_expired), 121 ENA_STAT_GLOBAL_ENTRY(interface_up), 122 ENA_STAT_GLOBAL_ENTRY(interface_down), 123 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 124 }; 125 126 static const struct ena_stats ena_stats_tx_strings[] = { 127 ENA_STAT_TX_ENTRY(cnt), 128 ENA_STAT_TX_ENTRY(bytes), 129 ENA_STAT_TX_ENTRY(queue_stop), 130 ENA_STAT_TX_ENTRY(queue_wakeup), 131 ENA_STAT_TX_ENTRY(dma_mapping_err), 132 ENA_STAT_TX_ENTRY(linearize), 133 ENA_STAT_TX_ENTRY(linearize_failed), 134 ENA_STAT_TX_ENTRY(tx_poll), 135 ENA_STAT_TX_ENTRY(doorbells), 136 ENA_STAT_TX_ENTRY(prepare_ctx_err), 137 ENA_STAT_TX_ENTRY(missing_tx_comp), 138 ENA_STAT_TX_ENTRY(bad_req_id), 139 }; 140 141 static const struct ena_stats ena_stats_rx_strings[] = { 142 ENA_STAT_RX_ENTRY(cnt), 143 ENA_STAT_RX_ENTRY(bytes), 144 ENA_STAT_RX_ENTRY(refil_partial), 145 ENA_STAT_RX_ENTRY(bad_csum), 146 ENA_STAT_RX_ENTRY(page_alloc_fail), 147 ENA_STAT_RX_ENTRY(skb_alloc_fail), 148 ENA_STAT_RX_ENTRY(dma_mapping_err), 149 ENA_STAT_RX_ENTRY(bad_desc_num), 150 ENA_STAT_RX_ENTRY(small_copy_len_pkt), 151 }; 152 153 static const struct ena_stats ena_stats_ena_com_strings[] = { 154 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 155 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 156 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 157 ENA_STAT_ENA_COM_ENTRY(out_of_space), 158 ENA_STAT_ENA_COM_ENTRY(no_completion), 159 }; 160 161 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 162 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 163 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 164 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 165 166 /** Vendor ID used by Amazon devices */ 167 #define PCI_VENDOR_ID_AMAZON 0x1D0F 168 /** Amazon devices */ 169 #define PCI_DEVICE_ID_ENA_VF 0xEC20 170 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 171 172 #define ENA_TX_OFFLOAD_MASK (\ 173 PKT_TX_L4_MASK | \ 174 PKT_TX_IP_CKSUM | \ 175 PKT_TX_TCP_SEG) 176 177 #define ENA_TX_OFFLOAD_NOTSUP_MASK \ 178 (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) 179 180 static const struct rte_pci_id pci_id_ena_map[] = { 181 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 182 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, 183 { .device_id = 0 }, 184 }; 185 186 static int ena_device_init(struct ena_com_dev *ena_dev, 187 struct ena_com_dev_get_features_ctx *get_feat_ctx); 188 static int ena_dev_configure(struct rte_eth_dev *dev); 189 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 190 uint16_t nb_pkts); 191 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 192 uint16_t nb_pkts); 193 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 194 uint16_t nb_desc, unsigned int socket_id, 195 const struct rte_eth_txconf *tx_conf); 196 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 197 uint16_t nb_desc, unsigned int socket_id, 198 const struct rte_eth_rxconf *rx_conf, 199 struct rte_mempool *mp); 200 static uint16_t eth_ena_recv_pkts(void *rx_queue, 201 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 202 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 203 static void ena_init_rings(struct ena_adapter *adapter); 204 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 205 static int ena_start(struct rte_eth_dev *dev); 206 static void ena_close(struct rte_eth_dev *dev); 207 static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 208 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 209 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 210 static void ena_rx_queue_release(void *queue); 211 static void ena_tx_queue_release(void *queue); 212 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 213 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 214 static int ena_link_update(struct rte_eth_dev *dev, 215 __rte_unused int wait_to_complete); 216 static int ena_queue_restart(struct ena_ring *ring); 217 static int ena_queue_restart_all(struct rte_eth_dev *dev, 218 enum ena_ring_type ring_type); 219 static void ena_stats_restart(struct rte_eth_dev *dev); 220 static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, 221 struct rte_eth_dev_info *dev_info); 222 static int ena_rss_reta_update(struct rte_eth_dev *dev, 223 struct rte_eth_rss_reta_entry64 *reta_conf, 224 uint16_t reta_size); 225 static int ena_rss_reta_query(struct rte_eth_dev *dev, 226 struct rte_eth_rss_reta_entry64 *reta_conf, 227 uint16_t reta_size); 228 static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); 229 230 static const struct eth_dev_ops ena_dev_ops = { 231 .dev_configure = ena_dev_configure, 232 .dev_infos_get = ena_infos_get, 233 .rx_queue_setup = ena_rx_queue_setup, 234 .tx_queue_setup = ena_tx_queue_setup, 235 .dev_start = ena_start, 236 .link_update = ena_link_update, 237 .stats_get = ena_stats_get, 238 .mtu_set = ena_mtu_set, 239 .rx_queue_release = ena_rx_queue_release, 240 .tx_queue_release = ena_tx_queue_release, 241 .dev_close = ena_close, 242 .reta_update = ena_rss_reta_update, 243 .reta_query = ena_rss_reta_query, 244 }; 245 246 #define NUMA_NO_NODE SOCKET_ID_ANY 247 248 static inline int ena_cpu_to_node(int cpu) 249 { 250 struct rte_config *config = rte_eal_get_configuration(); 251 252 if (likely(cpu < RTE_MAX_MEMZONE)) 253 return config->mem_config->memzone[cpu].socket_id; 254 255 return NUMA_NO_NODE; 256 } 257 258 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 259 struct ena_com_rx_ctx *ena_rx_ctx) 260 { 261 uint64_t ol_flags = 0; 262 263 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 264 ol_flags |= PKT_TX_TCP_CKSUM; 265 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 266 ol_flags |= PKT_TX_UDP_CKSUM; 267 268 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 269 ol_flags |= PKT_TX_IPV4; 270 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 271 ol_flags |= PKT_TX_IPV6; 272 273 if (unlikely(ena_rx_ctx->l4_csum_err)) 274 ol_flags |= PKT_RX_L4_CKSUM_BAD; 275 if (unlikely(ena_rx_ctx->l3_csum_err)) 276 ol_flags |= PKT_RX_IP_CKSUM_BAD; 277 278 mbuf->ol_flags = ol_flags; 279 } 280 281 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 282 struct ena_com_tx_ctx *ena_tx_ctx) 283 { 284 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 285 286 if (mbuf->ol_flags & 287 (PKT_TX_L4_MASK | PKT_TX_IP_CKSUM | PKT_TX_TCP_SEG)) { 288 /* check if TSO is required */ 289 if (mbuf->ol_flags & PKT_TX_TCP_SEG) { 290 ena_tx_ctx->tso_enable = true; 291 292 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 293 } 294 295 /* check if L3 checksum is needed */ 296 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 297 ena_tx_ctx->l3_csum_enable = true; 298 299 if (mbuf->ol_flags & PKT_TX_IPV6) { 300 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 301 } else { 302 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 303 304 /* set don't fragment (DF) flag */ 305 if (mbuf->packet_type & 306 (RTE_PTYPE_L4_NONFRAG 307 | RTE_PTYPE_INNER_L4_NONFRAG)) 308 ena_tx_ctx->df = true; 309 } 310 311 /* check if L4 checksum is needed */ 312 switch (mbuf->ol_flags & PKT_TX_L4_MASK) { 313 case PKT_TX_TCP_CKSUM: 314 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 315 ena_tx_ctx->l4_csum_enable = true; 316 break; 317 case PKT_TX_UDP_CKSUM: 318 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 319 ena_tx_ctx->l4_csum_enable = true; 320 break; 321 default: 322 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 323 ena_tx_ctx->l4_csum_enable = false; 324 break; 325 } 326 327 ena_meta->mss = mbuf->tso_segsz; 328 ena_meta->l3_hdr_len = mbuf->l3_len; 329 ena_meta->l3_hdr_offset = mbuf->l2_len; 330 /* this param needed only for TSO */ 331 ena_meta->l3_outer_hdr_len = 0; 332 ena_meta->l3_outer_hdr_offset = 0; 333 334 ena_tx_ctx->meta_valid = true; 335 } else { 336 ena_tx_ctx->meta_valid = false; 337 } 338 } 339 340 static void ena_config_host_info(struct ena_com_dev *ena_dev) 341 { 342 struct ena_admin_host_info *host_info; 343 int rc; 344 345 /* Allocate only the host info */ 346 rc = ena_com_allocate_host_info(ena_dev); 347 if (rc) { 348 RTE_LOG(ERR, PMD, "Cannot allocate host info\n"); 349 return; 350 } 351 352 host_info = ena_dev->host_attr.host_info; 353 354 host_info->os_type = ENA_ADMIN_OS_DPDK; 355 host_info->kernel_ver = RTE_VERSION; 356 snprintf((char *)host_info->kernel_ver_str, 357 sizeof(host_info->kernel_ver_str), 358 "%s", rte_version()); 359 host_info->os_dist = RTE_VERSION; 360 snprintf((char *)host_info->os_dist_str, 361 sizeof(host_info->os_dist_str), 362 "%s", rte_version()); 363 host_info->driver_version = 364 (DRV_MODULE_VER_MAJOR) | 365 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 366 (DRV_MODULE_VER_SUBMINOR << 367 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 368 369 rc = ena_com_set_host_attributes(ena_dev); 370 if (rc) { 371 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 372 if (rc != -EPERM) 373 goto err; 374 } 375 376 return; 377 378 err: 379 ena_com_delete_host_info(ena_dev); 380 } 381 382 static int 383 ena_get_sset_count(struct rte_eth_dev *dev, int sset) 384 { 385 if (sset != ETH_SS_STATS) 386 return -EOPNOTSUPP; 387 388 /* Workaround for clang: 389 * touch internal structures to prevent 390 * compiler error 391 */ 392 ENA_TOUCH(ena_stats_global_strings); 393 ENA_TOUCH(ena_stats_tx_strings); 394 ENA_TOUCH(ena_stats_rx_strings); 395 ENA_TOUCH(ena_stats_ena_com_strings); 396 397 return dev->data->nb_tx_queues * 398 (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) + 399 ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 400 } 401 402 static void ena_config_debug_area(struct ena_adapter *adapter) 403 { 404 u32 debug_area_size; 405 int rc, ss_count; 406 407 ss_count = ena_get_sset_count(adapter->rte_dev, ETH_SS_STATS); 408 if (ss_count <= 0) { 409 RTE_LOG(ERR, PMD, "SS count is negative\n"); 410 return; 411 } 412 413 /* allocate 32 bytes for each string and 64bit for the value */ 414 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 415 416 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 417 if (rc) { 418 RTE_LOG(ERR, PMD, "Cannot allocate debug area\n"); 419 return; 420 } 421 422 rc = ena_com_set_host_attributes(&adapter->ena_dev); 423 if (rc) { 424 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 425 if (rc != -EPERM) 426 goto err; 427 } 428 429 return; 430 err: 431 ena_com_delete_debug_area(&adapter->ena_dev); 432 } 433 434 static void ena_close(struct rte_eth_dev *dev) 435 { 436 struct ena_adapter *adapter = 437 (struct ena_adapter *)(dev->data->dev_private); 438 439 adapter->state = ENA_ADAPTER_STATE_STOPPED; 440 441 ena_rx_queue_release_all(dev); 442 ena_tx_queue_release_all(dev); 443 } 444 445 static int ena_rss_reta_update(struct rte_eth_dev *dev, 446 struct rte_eth_rss_reta_entry64 *reta_conf, 447 uint16_t reta_size) 448 { 449 struct ena_adapter *adapter = 450 (struct ena_adapter *)(dev->data->dev_private); 451 struct ena_com_dev *ena_dev = &adapter->ena_dev; 452 int ret, i; 453 u16 entry_value; 454 int conf_idx; 455 int idx; 456 457 if ((reta_size == 0) || (reta_conf == NULL)) 458 return -EINVAL; 459 460 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 461 RTE_LOG(WARNING, PMD, 462 "indirection table %d is bigger than supported (%d)\n", 463 reta_size, ENA_RX_RSS_TABLE_SIZE); 464 ret = -EINVAL; 465 goto err; 466 } 467 468 for (i = 0 ; i < reta_size ; i++) { 469 /* each reta_conf is for 64 entries. 470 * to support 128 we use 2 conf of 64 471 */ 472 conf_idx = i / RTE_RETA_GROUP_SIZE; 473 idx = i % RTE_RETA_GROUP_SIZE; 474 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 475 entry_value = 476 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 477 ret = ena_com_indirect_table_fill_entry(ena_dev, 478 i, 479 entry_value); 480 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 481 RTE_LOG(ERR, PMD, 482 "Cannot fill indirect table\n"); 483 ret = -ENOTSUP; 484 goto err; 485 } 486 } 487 } 488 489 ret = ena_com_indirect_table_set(ena_dev); 490 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 491 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 492 ret = -ENOTSUP; 493 goto err; 494 } 495 496 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 497 __func__, reta_size, adapter->rte_dev->data->port_id); 498 err: 499 return ret; 500 } 501 502 /* Query redirection table. */ 503 static int ena_rss_reta_query(struct rte_eth_dev *dev, 504 struct rte_eth_rss_reta_entry64 *reta_conf, 505 uint16_t reta_size) 506 { 507 struct ena_adapter *adapter = 508 (struct ena_adapter *)(dev->data->dev_private); 509 struct ena_com_dev *ena_dev = &adapter->ena_dev; 510 int ret; 511 int i; 512 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 513 int reta_conf_idx; 514 int reta_idx; 515 516 if (reta_size == 0 || reta_conf == NULL || 517 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 518 return -EINVAL; 519 520 ret = ena_com_indirect_table_get(ena_dev, indirect_table); 521 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 522 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 523 ret = -ENOTSUP; 524 goto err; 525 } 526 527 for (i = 0 ; i < reta_size ; i++) { 528 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 529 reta_idx = i % RTE_RETA_GROUP_SIZE; 530 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 531 reta_conf[reta_conf_idx].reta[reta_idx] = 532 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 533 } 534 err: 535 return ret; 536 } 537 538 static int ena_rss_init_default(struct ena_adapter *adapter) 539 { 540 struct ena_com_dev *ena_dev = &adapter->ena_dev; 541 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 542 int rc, i; 543 u32 val; 544 545 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 546 if (unlikely(rc)) { 547 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 548 goto err_rss_init; 549 } 550 551 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 552 val = i % nb_rx_queues; 553 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 554 ENA_IO_RXQ_IDX(val)); 555 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 556 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 557 goto err_fill_indir; 558 } 559 } 560 561 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 562 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 563 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 564 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 565 goto err_fill_indir; 566 } 567 568 rc = ena_com_set_default_hash_ctrl(ena_dev); 569 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 570 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 571 goto err_fill_indir; 572 } 573 574 rc = ena_com_indirect_table_set(ena_dev); 575 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 576 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 577 goto err_fill_indir; 578 } 579 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 580 adapter->rte_dev->data->port_id); 581 582 return 0; 583 584 err_fill_indir: 585 ena_com_rss_destroy(ena_dev); 586 err_rss_init: 587 588 return rc; 589 } 590 591 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 592 { 593 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 594 int nb_queues = dev->data->nb_rx_queues; 595 int i; 596 597 for (i = 0; i < nb_queues; i++) 598 ena_rx_queue_release(queues[i]); 599 } 600 601 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 602 { 603 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 604 int nb_queues = dev->data->nb_tx_queues; 605 int i; 606 607 for (i = 0; i < nb_queues; i++) 608 ena_tx_queue_release(queues[i]); 609 } 610 611 static void ena_rx_queue_release(void *queue) 612 { 613 struct ena_ring *ring = (struct ena_ring *)queue; 614 struct ena_adapter *adapter = ring->adapter; 615 int ena_qid; 616 617 ena_assert_msg(ring->configured, 618 "API violation - releasing not configured queue"); 619 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 620 "API violation"); 621 622 /* Destroy HW queue */ 623 ena_qid = ENA_IO_RXQ_IDX(ring->id); 624 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 625 626 /* Free all bufs */ 627 ena_rx_queue_release_bufs(ring); 628 629 /* Free ring resources */ 630 if (ring->rx_buffer_info) 631 rte_free(ring->rx_buffer_info); 632 ring->rx_buffer_info = NULL; 633 634 ring->configured = 0; 635 636 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 637 ring->port_id, ring->id); 638 } 639 640 static void ena_tx_queue_release(void *queue) 641 { 642 struct ena_ring *ring = (struct ena_ring *)queue; 643 struct ena_adapter *adapter = ring->adapter; 644 int ena_qid; 645 646 ena_assert_msg(ring->configured, 647 "API violation. Releasing not configured queue"); 648 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 649 "API violation"); 650 651 /* Destroy HW queue */ 652 ena_qid = ENA_IO_TXQ_IDX(ring->id); 653 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 654 655 /* Free all bufs */ 656 ena_tx_queue_release_bufs(ring); 657 658 /* Free ring resources */ 659 if (ring->tx_buffer_info) 660 rte_free(ring->tx_buffer_info); 661 662 if (ring->empty_tx_reqs) 663 rte_free(ring->empty_tx_reqs); 664 665 ring->empty_tx_reqs = NULL; 666 ring->tx_buffer_info = NULL; 667 668 ring->configured = 0; 669 670 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 671 ring->port_id, ring->id); 672 } 673 674 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 675 { 676 unsigned int ring_mask = ring->ring_size - 1; 677 678 while (ring->next_to_clean != ring->next_to_use) { 679 struct rte_mbuf *m = 680 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 681 682 if (m) 683 __rte_mbuf_raw_free(m); 684 685 ring->next_to_clean++; 686 } 687 } 688 689 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 690 { 691 unsigned int ring_mask = ring->ring_size - 1; 692 693 while (ring->next_to_clean != ring->next_to_use) { 694 struct ena_tx_buffer *tx_buf = 695 &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; 696 697 if (tx_buf->mbuf) 698 rte_pktmbuf_free(tx_buf->mbuf); 699 700 ring->next_to_clean++; 701 } 702 } 703 704 static int ena_link_update(struct rte_eth_dev *dev, 705 __rte_unused int wait_to_complete) 706 { 707 struct rte_eth_link *link = &dev->data->dev_link; 708 709 link->link_status = 1; 710 link->link_speed = ETH_SPEED_NUM_10G; 711 link->link_duplex = ETH_LINK_FULL_DUPLEX; 712 713 return 0; 714 } 715 716 static int ena_queue_restart_all(struct rte_eth_dev *dev, 717 enum ena_ring_type ring_type) 718 { 719 struct ena_adapter *adapter = 720 (struct ena_adapter *)(dev->data->dev_private); 721 struct ena_ring *queues = NULL; 722 int i = 0; 723 int rc = 0; 724 725 queues = (ring_type == ENA_RING_TYPE_RX) ? 726 adapter->rx_ring : adapter->tx_ring; 727 728 for (i = 0; i < adapter->num_queues; i++) { 729 if (queues[i].configured) { 730 if (ring_type == ENA_RING_TYPE_RX) { 731 ena_assert_msg( 732 dev->data->rx_queues[i] == &queues[i], 733 "Inconsistent state of rx queues\n"); 734 } else { 735 ena_assert_msg( 736 dev->data->tx_queues[i] == &queues[i], 737 "Inconsistent state of tx queues\n"); 738 } 739 740 rc = ena_queue_restart(&queues[i]); 741 742 if (rc) { 743 PMD_INIT_LOG(ERR, 744 "failed to restart queue %d type(%d)", 745 i, ring_type); 746 return -1; 747 } 748 } 749 } 750 751 return 0; 752 } 753 754 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 755 { 756 uint32_t max_frame_len = adapter->max_mtu; 757 758 if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) 759 max_frame_len = 760 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 761 762 return max_frame_len; 763 } 764 765 static int ena_check_valid_conf(struct ena_adapter *adapter) 766 { 767 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 768 769 if (max_frame_len > adapter->max_mtu) { 770 PMD_INIT_LOG(ERR, "Unsupported MTU of %d", max_frame_len); 771 return -1; 772 } 773 774 return 0; 775 } 776 777 static int 778 ena_calc_queue_size(struct ena_com_dev *ena_dev, 779 struct ena_com_dev_get_features_ctx *get_feat_ctx) 780 { 781 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 782 783 queue_size = RTE_MIN(queue_size, 784 get_feat_ctx->max_queues.max_cq_depth); 785 queue_size = RTE_MIN(queue_size, 786 get_feat_ctx->max_queues.max_sq_depth); 787 788 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 789 queue_size = RTE_MIN(queue_size, 790 get_feat_ctx->max_queues.max_llq_depth); 791 792 /* Round down to power of 2 */ 793 if (!rte_is_power_of_2(queue_size)) 794 queue_size = rte_align32pow2(queue_size >> 1); 795 796 if (queue_size == 0) { 797 PMD_INIT_LOG(ERR, "Invalid queue size"); 798 return -EFAULT; 799 } 800 801 return queue_size; 802 } 803 804 static void ena_stats_restart(struct rte_eth_dev *dev) 805 { 806 struct ena_adapter *adapter = 807 (struct ena_adapter *)(dev->data->dev_private); 808 809 rte_atomic64_init(&adapter->drv_stats->ierrors); 810 rte_atomic64_init(&adapter->drv_stats->oerrors); 811 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 812 } 813 814 static void ena_stats_get(struct rte_eth_dev *dev, 815 struct rte_eth_stats *stats) 816 { 817 struct ena_admin_basic_stats ena_stats; 818 struct ena_adapter *adapter = 819 (struct ena_adapter *)(dev->data->dev_private); 820 struct ena_com_dev *ena_dev = &adapter->ena_dev; 821 int rc; 822 823 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 824 return; 825 826 memset(&ena_stats, 0, sizeof(ena_stats)); 827 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 828 if (unlikely(rc)) { 829 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 830 return; 831 } 832 833 /* Set of basic statistics from ENA */ 834 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 835 ena_stats.rx_pkts_low); 836 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 837 ena_stats.tx_pkts_low); 838 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 839 ena_stats.rx_bytes_low); 840 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 841 ena_stats.tx_bytes_low); 842 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 843 ena_stats.rx_drops_low); 844 845 /* Driver related stats */ 846 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 847 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 848 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 849 } 850 851 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 852 { 853 struct ena_adapter *adapter; 854 struct ena_com_dev *ena_dev; 855 int rc = 0; 856 857 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 858 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 859 adapter = (struct ena_adapter *)(dev->data->dev_private); 860 861 ena_dev = &adapter->ena_dev; 862 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 863 864 if (mtu > ena_get_mtu_conf(adapter)) { 865 RTE_LOG(ERR, PMD, 866 "Given MTU (%d) exceeds maximum MTU supported (%d)\n", 867 mtu, ena_get_mtu_conf(adapter)); 868 rc = -EINVAL; 869 goto err; 870 } 871 872 rc = ena_com_set_dev_mtu(ena_dev, mtu); 873 if (rc) 874 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 875 else 876 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 877 878 err: 879 return rc; 880 } 881 882 static int ena_start(struct rte_eth_dev *dev) 883 { 884 struct ena_adapter *adapter = 885 (struct ena_adapter *)(dev->data->dev_private); 886 int rc = 0; 887 888 if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || 889 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 890 PMD_INIT_LOG(ERR, "API violation"); 891 return -1; 892 } 893 894 rc = ena_check_valid_conf(adapter); 895 if (rc) 896 return rc; 897 898 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 899 if (rc) 900 return rc; 901 902 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 903 if (rc) 904 return rc; 905 906 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 907 ETH_MQ_RX_RSS_FLAG) { 908 rc = ena_rss_init_default(adapter); 909 if (rc) 910 return rc; 911 } 912 913 ena_stats_restart(dev); 914 915 adapter->state = ENA_ADAPTER_STATE_RUNNING; 916 917 return 0; 918 } 919 920 static int ena_queue_restart(struct ena_ring *ring) 921 { 922 int rc; 923 924 ena_assert_msg(ring->configured == 1, 925 "Trying to restart unconfigured queue\n"); 926 927 ring->next_to_clean = 0; 928 ring->next_to_use = 0; 929 930 if (ring->type == ENA_RING_TYPE_TX) 931 return 0; 932 933 rc = ena_populate_rx_queue(ring, ring->ring_size); 934 if ((unsigned int)rc != ring->ring_size) { 935 PMD_INIT_LOG(ERR, "Failed to populate rx ring !"); 936 return (-1); 937 } 938 939 return 0; 940 } 941 942 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 943 uint16_t queue_idx, 944 uint16_t nb_desc, 945 __rte_unused unsigned int socket_id, 946 __rte_unused const struct rte_eth_txconf *tx_conf) 947 { 948 struct ena_com_create_io_ctx ctx = 949 /* policy set to _HOST just to satisfy icc compiler */ 950 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 951 ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 }; 952 struct ena_ring *txq = NULL; 953 struct ena_adapter *adapter = 954 (struct ena_adapter *)(dev->data->dev_private); 955 unsigned int i; 956 int ena_qid; 957 int rc; 958 struct ena_com_dev *ena_dev = &adapter->ena_dev; 959 960 txq = &adapter->tx_ring[queue_idx]; 961 962 if (txq->configured) { 963 RTE_LOG(CRIT, PMD, 964 "API violation. Queue %d is already configured\n", 965 queue_idx); 966 return -1; 967 } 968 969 if (!rte_is_power_of_2(nb_desc)) { 970 RTE_LOG(ERR, PMD, 971 "Unsupported size of RX queue: %d is not a power of 2.", 972 nb_desc); 973 return -EINVAL; 974 } 975 976 if (nb_desc > adapter->tx_ring_size) { 977 RTE_LOG(ERR, PMD, 978 "Unsupported size of TX queue (max size: %d)\n", 979 adapter->tx_ring_size); 980 return -EINVAL; 981 } 982 983 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 984 985 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 986 ctx.qid = ena_qid; 987 ctx.msix_vector = -1; /* admin interrupts not used */ 988 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 989 ctx.queue_size = adapter->tx_ring_size; 990 ctx.numa_node = ena_cpu_to_node(queue_idx); 991 992 rc = ena_com_create_io_queue(ena_dev, &ctx); 993 if (rc) { 994 RTE_LOG(ERR, PMD, 995 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 996 queue_idx, ena_qid, rc); 997 } 998 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 999 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1000 1001 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1002 &txq->ena_com_io_sq, 1003 &txq->ena_com_io_cq); 1004 if (rc) { 1005 RTE_LOG(ERR, PMD, 1006 "Failed to get TX queue handlers. TX queue num %d rc: %d\n", 1007 queue_idx, rc); 1008 ena_com_destroy_io_queue(ena_dev, ena_qid); 1009 goto err; 1010 } 1011 1012 txq->port_id = dev->data->port_id; 1013 txq->next_to_clean = 0; 1014 txq->next_to_use = 0; 1015 txq->ring_size = nb_desc; 1016 1017 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1018 sizeof(struct ena_tx_buffer) * 1019 txq->ring_size, 1020 RTE_CACHE_LINE_SIZE); 1021 if (!txq->tx_buffer_info) { 1022 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 1023 return -ENOMEM; 1024 } 1025 1026 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1027 sizeof(u16) * txq->ring_size, 1028 RTE_CACHE_LINE_SIZE); 1029 if (!txq->empty_tx_reqs) { 1030 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 1031 rte_free(txq->tx_buffer_info); 1032 return -ENOMEM; 1033 } 1034 for (i = 0; i < txq->ring_size; i++) 1035 txq->empty_tx_reqs[i] = i; 1036 1037 /* Store pointer to this queue in upper layer */ 1038 txq->configured = 1; 1039 dev->data->tx_queues[queue_idx] = txq; 1040 err: 1041 return rc; 1042 } 1043 1044 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1045 uint16_t queue_idx, 1046 uint16_t nb_desc, 1047 __rte_unused unsigned int socket_id, 1048 __rte_unused const struct rte_eth_rxconf *rx_conf, 1049 struct rte_mempool *mp) 1050 { 1051 struct ena_com_create_io_ctx ctx = 1052 /* policy set to _HOST just to satisfy icc compiler */ 1053 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1054 ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 }; 1055 struct ena_adapter *adapter = 1056 (struct ena_adapter *)(dev->data->dev_private); 1057 struct ena_ring *rxq = NULL; 1058 uint16_t ena_qid = 0; 1059 int rc = 0; 1060 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1061 1062 rxq = &adapter->rx_ring[queue_idx]; 1063 if (rxq->configured) { 1064 RTE_LOG(CRIT, PMD, 1065 "API violation. Queue %d is already configured\n", 1066 queue_idx); 1067 return -1; 1068 } 1069 1070 if (!rte_is_power_of_2(nb_desc)) { 1071 RTE_LOG(ERR, PMD, 1072 "Unsupported size of TX queue: %d is not a power of 2.", 1073 nb_desc); 1074 return -EINVAL; 1075 } 1076 1077 if (nb_desc > adapter->rx_ring_size) { 1078 RTE_LOG(ERR, PMD, 1079 "Unsupported size of RX queue (max size: %d)\n", 1080 adapter->rx_ring_size); 1081 return -EINVAL; 1082 } 1083 1084 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 1085 1086 ctx.qid = ena_qid; 1087 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1088 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1089 ctx.msix_vector = -1; /* admin interrupts not used */ 1090 ctx.queue_size = adapter->rx_ring_size; 1091 ctx.numa_node = ena_cpu_to_node(queue_idx); 1092 1093 rc = ena_com_create_io_queue(ena_dev, &ctx); 1094 if (rc) 1095 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 1096 queue_idx, rc); 1097 1098 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 1099 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1100 1101 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1102 &rxq->ena_com_io_sq, 1103 &rxq->ena_com_io_cq); 1104 if (rc) { 1105 RTE_LOG(ERR, PMD, 1106 "Failed to get RX queue handlers. RX queue num %d rc: %d\n", 1107 queue_idx, rc); 1108 ena_com_destroy_io_queue(ena_dev, ena_qid); 1109 } 1110 1111 rxq->port_id = dev->data->port_id; 1112 rxq->next_to_clean = 0; 1113 rxq->next_to_use = 0; 1114 rxq->ring_size = nb_desc; 1115 rxq->mb_pool = mp; 1116 1117 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1118 sizeof(struct rte_mbuf *) * nb_desc, 1119 RTE_CACHE_LINE_SIZE); 1120 if (!rxq->rx_buffer_info) { 1121 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 1122 return -ENOMEM; 1123 } 1124 1125 /* Store pointer to this queue in upper layer */ 1126 rxq->configured = 1; 1127 dev->data->rx_queues[queue_idx] = rxq; 1128 1129 return rc; 1130 } 1131 1132 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1133 { 1134 unsigned int i; 1135 int rc; 1136 uint16_t ring_size = rxq->ring_size; 1137 uint16_t ring_mask = ring_size - 1; 1138 uint16_t next_to_use = rxq->next_to_use; 1139 uint16_t in_use; 1140 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 1141 1142 if (unlikely(!count)) 1143 return 0; 1144 1145 in_use = rxq->next_to_use - rxq->next_to_clean; 1146 ena_assert_msg(((in_use + count) <= ring_size), "bad ring state"); 1147 1148 count = RTE_MIN(count, 1149 (uint16_t)(ring_size - (next_to_use & ring_mask))); 1150 1151 /* get resources for incoming packets */ 1152 rc = rte_mempool_get_bulk(rxq->mb_pool, 1153 (void **)(&mbufs[next_to_use & ring_mask]), 1154 count); 1155 if (unlikely(rc < 0)) { 1156 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1157 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1158 return 0; 1159 } 1160 1161 for (i = 0; i < count; i++) { 1162 uint16_t next_to_use_masked = next_to_use & ring_mask; 1163 struct rte_mbuf *mbuf = mbufs[next_to_use_masked]; 1164 struct ena_com_buf ebuf; 1165 1166 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 1167 /* prepare physical address for DMA transaction */ 1168 ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; 1169 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1170 /* pass resource to device */ 1171 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 1172 &ebuf, next_to_use_masked); 1173 if (unlikely(rc)) { 1174 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 1175 break; 1176 } 1177 next_to_use++; 1178 } 1179 1180 /* When we submitted free recources to device... */ 1181 if (i > 0) { 1182 /* ...let HW know that it can fill buffers with data */ 1183 rte_wmb(); 1184 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1185 1186 rxq->next_to_use = next_to_use; 1187 } 1188 1189 return i; 1190 } 1191 1192 static int ena_device_init(struct ena_com_dev *ena_dev, 1193 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1194 { 1195 int rc; 1196 bool readless_supported; 1197 1198 /* Initialize mmio registers */ 1199 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1200 if (rc) { 1201 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 1202 return rc; 1203 } 1204 1205 /* The PCIe configuration space revision id indicate if mmio reg 1206 * read is disabled. 1207 */ 1208 readless_supported = 1209 !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id 1210 & ENA_MMIO_DISABLE_REG_READ); 1211 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1212 1213 /* reset device */ 1214 rc = ena_com_dev_reset(ena_dev); 1215 if (rc) { 1216 RTE_LOG(ERR, PMD, "cannot reset device\n"); 1217 goto err_mmio_read_less; 1218 } 1219 1220 /* check FW version */ 1221 rc = ena_com_validate_version(ena_dev); 1222 if (rc) { 1223 RTE_LOG(ERR, PMD, "device version is too low\n"); 1224 goto err_mmio_read_less; 1225 } 1226 1227 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1228 1229 /* ENA device administration layer init */ 1230 rc = ena_com_admin_init(ena_dev, NULL, true); 1231 if (rc) { 1232 RTE_LOG(ERR, PMD, 1233 "cannot initialize ena admin queue with device\n"); 1234 goto err_mmio_read_less; 1235 } 1236 1237 /* To enable the msix interrupts the driver needs to know the number 1238 * of queues. So the driver uses polling mode to retrieve this 1239 * information. 1240 */ 1241 ena_com_set_admin_polling_mode(ena_dev, true); 1242 1243 ena_config_host_info(ena_dev); 1244 1245 /* Get Device Attributes and features */ 1246 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1247 if (rc) { 1248 RTE_LOG(ERR, PMD, 1249 "cannot get attribute for ena device rc= %d\n", rc); 1250 goto err_admin_init; 1251 } 1252 1253 return 0; 1254 1255 err_admin_init: 1256 ena_com_admin_destroy(ena_dev); 1257 1258 err_mmio_read_less: 1259 ena_com_mmio_reg_read_request_destroy(ena_dev); 1260 1261 return rc; 1262 } 1263 1264 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1265 { 1266 struct rte_pci_device *pci_dev; 1267 struct ena_adapter *adapter = 1268 (struct ena_adapter *)(eth_dev->data->dev_private); 1269 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1270 struct ena_com_dev_get_features_ctx get_feat_ctx; 1271 int queue_size, rc; 1272 1273 static int adapters_found; 1274 1275 memset(adapter, 0, sizeof(struct ena_adapter)); 1276 ena_dev = &adapter->ena_dev; 1277 1278 eth_dev->dev_ops = &ena_dev_ops; 1279 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1280 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1281 eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; 1282 adapter->rte_eth_dev_data = eth_dev->data; 1283 adapter->rte_dev = eth_dev; 1284 1285 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1286 return 0; 1287 1288 pci_dev = RTE_DEV_TO_PCI(eth_dev->device); 1289 adapter->pdev = pci_dev; 1290 1291 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", 1292 pci_dev->addr.domain, 1293 pci_dev->addr.bus, 1294 pci_dev->addr.devid, 1295 pci_dev->addr.function); 1296 1297 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1298 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1299 1300 /* Present ENA_MEM_BAR indicates available LLQ mode. 1301 * Use corresponding policy 1302 */ 1303 if (adapter->dev_mem_base) 1304 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 1305 else if (adapter->regs) 1306 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1307 else 1308 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)", 1309 ENA_REGS_BAR); 1310 1311 ena_dev->reg_bar = adapter->regs; 1312 ena_dev->dmadev = adapter->pdev; 1313 1314 adapter->id_number = adapters_found; 1315 1316 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1317 adapter->id_number); 1318 1319 /* device specific initialization routine */ 1320 rc = ena_device_init(ena_dev, &get_feat_ctx); 1321 if (rc) { 1322 PMD_INIT_LOG(CRIT, "Failed to init ENA device"); 1323 return -1; 1324 } 1325 1326 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1327 if (get_feat_ctx.max_queues.max_llq_num == 0) { 1328 PMD_INIT_LOG(ERR, 1329 "Trying to use LLQ but llq_num is 0.\n" 1330 "Fall back into regular queues."); 1331 ena_dev->tx_mem_queue_type = 1332 ENA_ADMIN_PLACEMENT_POLICY_HOST; 1333 adapter->num_queues = 1334 get_feat_ctx.max_queues.max_sq_num; 1335 } else { 1336 adapter->num_queues = 1337 get_feat_ctx.max_queues.max_llq_num; 1338 } 1339 } else { 1340 adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; 1341 } 1342 1343 queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); 1344 if ((queue_size <= 0) || (adapter->num_queues <= 0)) 1345 return -EFAULT; 1346 1347 adapter->tx_ring_size = queue_size; 1348 adapter->rx_ring_size = queue_size; 1349 1350 /* prepare ring structures */ 1351 ena_init_rings(adapter); 1352 1353 ena_config_debug_area(adapter); 1354 1355 /* Set max MTU for this device */ 1356 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1357 1358 /* set device support for TSO */ 1359 adapter->tso4_supported = get_feat_ctx.offload.tx & 1360 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; 1361 1362 /* Copy MAC address and point DPDK to it */ 1363 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1364 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1365 (struct ether_addr *)adapter->mac_addr); 1366 1367 adapter->drv_stats = rte_zmalloc("adapter stats", 1368 sizeof(*adapter->drv_stats), 1369 RTE_CACHE_LINE_SIZE); 1370 if (!adapter->drv_stats) { 1371 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1372 return -ENOMEM; 1373 } 1374 1375 adapters_found++; 1376 adapter->state = ENA_ADAPTER_STATE_INIT; 1377 1378 return 0; 1379 } 1380 1381 static int ena_dev_configure(struct rte_eth_dev *dev) 1382 { 1383 struct ena_adapter *adapter = 1384 (struct ena_adapter *)(dev->data->dev_private); 1385 1386 if (!(adapter->state == ENA_ADAPTER_STATE_INIT || 1387 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 1388 PMD_INIT_LOG(ERR, "Illegal adapter state: %d", 1389 adapter->state); 1390 return -1; 1391 } 1392 1393 switch (adapter->state) { 1394 case ENA_ADAPTER_STATE_INIT: 1395 case ENA_ADAPTER_STATE_STOPPED: 1396 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1397 break; 1398 case ENA_ADAPTER_STATE_CONFIG: 1399 RTE_LOG(WARNING, PMD, 1400 "Ivalid driver state while trying to configure device\n"); 1401 break; 1402 default: 1403 break; 1404 } 1405 1406 return 0; 1407 } 1408 1409 static void ena_init_rings(struct ena_adapter *adapter) 1410 { 1411 int i; 1412 1413 for (i = 0; i < adapter->num_queues; i++) { 1414 struct ena_ring *ring = &adapter->tx_ring[i]; 1415 1416 ring->configured = 0; 1417 ring->type = ENA_RING_TYPE_TX; 1418 ring->adapter = adapter; 1419 ring->id = i; 1420 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1421 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1422 } 1423 1424 for (i = 0; i < adapter->num_queues; i++) { 1425 struct ena_ring *ring = &adapter->rx_ring[i]; 1426 1427 ring->configured = 0; 1428 ring->type = ENA_RING_TYPE_RX; 1429 ring->adapter = adapter; 1430 ring->id = i; 1431 } 1432 } 1433 1434 static void ena_infos_get(struct rte_eth_dev *dev, 1435 struct rte_eth_dev_info *dev_info) 1436 { 1437 struct ena_adapter *adapter; 1438 struct ena_com_dev *ena_dev; 1439 struct ena_com_dev_get_features_ctx feat; 1440 uint32_t rx_feat = 0, tx_feat = 0; 1441 int rc = 0; 1442 1443 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1444 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1445 adapter = (struct ena_adapter *)(dev->data->dev_private); 1446 1447 ena_dev = &adapter->ena_dev; 1448 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1449 1450 dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); 1451 1452 dev_info->speed_capa = 1453 ETH_LINK_SPEED_1G | 1454 ETH_LINK_SPEED_2_5G | 1455 ETH_LINK_SPEED_5G | 1456 ETH_LINK_SPEED_10G | 1457 ETH_LINK_SPEED_25G | 1458 ETH_LINK_SPEED_40G | 1459 ETH_LINK_SPEED_50G | 1460 ETH_LINK_SPEED_100G; 1461 1462 /* Get supported features from HW */ 1463 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1464 if (unlikely(rc)) { 1465 RTE_LOG(ERR, PMD, 1466 "Cannot get attribute for ena device rc= %d\n", rc); 1467 return; 1468 } 1469 1470 /* Set Tx & Rx features available for device */ 1471 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1472 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1473 1474 if (feat.offload.tx & 1475 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1476 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1477 DEV_TX_OFFLOAD_UDP_CKSUM | 1478 DEV_TX_OFFLOAD_TCP_CKSUM; 1479 1480 if (feat.offload.rx_supported & 1481 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1482 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1483 DEV_RX_OFFLOAD_UDP_CKSUM | 1484 DEV_RX_OFFLOAD_TCP_CKSUM; 1485 1486 /* Inform framework about available features */ 1487 dev_info->rx_offload_capa = rx_feat; 1488 dev_info->tx_offload_capa = tx_feat; 1489 1490 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1491 dev_info->max_rx_pktlen = adapter->max_mtu; 1492 dev_info->max_mac_addrs = 1; 1493 1494 dev_info->max_rx_queues = adapter->num_queues; 1495 dev_info->max_tx_queues = adapter->num_queues; 1496 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1497 } 1498 1499 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1500 uint16_t nb_pkts) 1501 { 1502 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1503 unsigned int ring_size = rx_ring->ring_size; 1504 unsigned int ring_mask = ring_size - 1; 1505 uint16_t next_to_clean = rx_ring->next_to_clean; 1506 uint16_t desc_in_use = 0; 1507 unsigned int recv_idx = 0; 1508 struct rte_mbuf *mbuf = NULL; 1509 struct rte_mbuf *mbuf_head = NULL; 1510 struct rte_mbuf *mbuf_prev = NULL; 1511 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1512 unsigned int completed; 1513 1514 struct ena_com_rx_ctx ena_rx_ctx; 1515 int rc = 0; 1516 1517 /* Check adapter state */ 1518 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1519 RTE_LOG(ALERT, PMD, 1520 "Trying to receive pkts while device is NOT running\n"); 1521 return 0; 1522 } 1523 1524 desc_in_use = rx_ring->next_to_use - next_to_clean; 1525 if (unlikely(nb_pkts > desc_in_use)) 1526 nb_pkts = desc_in_use; 1527 1528 for (completed = 0; completed < nb_pkts; completed++) { 1529 int segments = 0; 1530 1531 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1532 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1533 ena_rx_ctx.descs = 0; 1534 /* receive packet context */ 1535 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1536 rx_ring->ena_com_io_sq, 1537 &ena_rx_ctx); 1538 if (unlikely(rc)) { 1539 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1540 return 0; 1541 } 1542 1543 if (unlikely(ena_rx_ctx.descs == 0)) 1544 break; 1545 1546 while (segments < ena_rx_ctx.descs) { 1547 mbuf = rx_buff_info[next_to_clean & ring_mask]; 1548 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1549 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1550 mbuf->refcnt = 1; 1551 mbuf->next = NULL; 1552 if (segments == 0) { 1553 mbuf->nb_segs = ena_rx_ctx.descs; 1554 mbuf->port = rx_ring->port_id; 1555 mbuf->pkt_len = 0; 1556 mbuf_head = mbuf; 1557 } else { 1558 /* for multi-segment pkts create mbuf chain */ 1559 mbuf_prev->next = mbuf; 1560 } 1561 mbuf_head->pkt_len += mbuf->data_len; 1562 1563 mbuf_prev = mbuf; 1564 segments++; 1565 next_to_clean++; 1566 } 1567 1568 /* fill mbuf attributes if any */ 1569 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1570 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1571 1572 /* pass to DPDK application head mbuf */ 1573 rx_pkts[recv_idx] = mbuf_head; 1574 recv_idx++; 1575 } 1576 1577 /* Burst refill to save doorbells, memory barriers, const interval */ 1578 if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) 1579 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use); 1580 1581 rx_ring->next_to_clean = next_to_clean; 1582 1583 return recv_idx; 1584 } 1585 1586 static uint16_t 1587 eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1588 uint16_t nb_pkts) 1589 { 1590 int32_t ret; 1591 uint32_t i; 1592 struct rte_mbuf *m; 1593 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1594 struct ipv4_hdr *ip_hdr; 1595 uint64_t ol_flags; 1596 uint16_t frag_field; 1597 1598 /* ENA needs partial checksum for TSO packets only, skip early */ 1599 if (!tx_ring->adapter->tso4_supported) 1600 return nb_pkts; 1601 1602 for (i = 0; i != nb_pkts; i++) { 1603 m = tx_pkts[i]; 1604 ol_flags = m->ol_flags; 1605 1606 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || 1607 (ol_flags & PKT_TX_L4_MASK) == 1608 PKT_TX_SCTP_CKSUM) { 1609 rte_errno = -ENOTSUP; 1610 return i; 1611 } 1612 1613 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1614 ret = rte_validate_tx_offload(m); 1615 if (ret != 0) { 1616 rte_errno = ret; 1617 return i; 1618 } 1619 #endif 1620 1621 if (!(m->ol_flags & PKT_TX_IPV4)) 1622 continue; 1623 1624 ip_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, 1625 m->l2_len); 1626 frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); 1627 if (frag_field & IPV4_HDR_DF_FLAG) 1628 continue; 1629 1630 /* In case we are supposed to TSO and have DF not set (DF=0) 1631 * hardware must be provided with partial checksum, otherwise 1632 * it will take care of necessary calculations. 1633 */ 1634 1635 ret = rte_net_intel_cksum_flags_prepare(m, 1636 ol_flags & ~PKT_TX_TCP_SEG); 1637 if (ret != 0) { 1638 rte_errno = ret; 1639 return i; 1640 } 1641 } 1642 1643 return i; 1644 } 1645 1646 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1647 uint16_t nb_pkts) 1648 { 1649 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1650 uint16_t next_to_use = tx_ring->next_to_use; 1651 uint16_t next_to_clean = tx_ring->next_to_clean; 1652 struct rte_mbuf *mbuf; 1653 unsigned int ring_size = tx_ring->ring_size; 1654 unsigned int ring_mask = ring_size - 1; 1655 struct ena_com_tx_ctx ena_tx_ctx; 1656 struct ena_tx_buffer *tx_info; 1657 struct ena_com_buf *ebuf; 1658 uint16_t rc, req_id, total_tx_descs = 0; 1659 uint16_t sent_idx = 0, empty_tx_reqs; 1660 int nb_hw_desc; 1661 1662 /* Check adapter state */ 1663 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1664 RTE_LOG(ALERT, PMD, 1665 "Trying to xmit pkts while device is NOT running\n"); 1666 return 0; 1667 } 1668 1669 empty_tx_reqs = ring_size - (next_to_use - next_to_clean); 1670 if (nb_pkts > empty_tx_reqs) 1671 nb_pkts = empty_tx_reqs; 1672 1673 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 1674 mbuf = tx_pkts[sent_idx]; 1675 1676 req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; 1677 tx_info = &tx_ring->tx_buffer_info[req_id]; 1678 tx_info->mbuf = mbuf; 1679 tx_info->num_of_bufs = 0; 1680 ebuf = tx_info->bufs; 1681 1682 /* Prepare TX context */ 1683 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1684 memset(&ena_tx_ctx.ena_meta, 0x0, 1685 sizeof(struct ena_com_tx_meta)); 1686 ena_tx_ctx.ena_bufs = ebuf; 1687 ena_tx_ctx.req_id = req_id; 1688 if (tx_ring->tx_mem_queue_type == 1689 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1690 /* prepare the push buffer with 1691 * virtual address of the data 1692 */ 1693 ena_tx_ctx.header_len = 1694 RTE_MIN(mbuf->data_len, 1695 tx_ring->tx_max_header_size); 1696 ena_tx_ctx.push_header = 1697 (void *)((char *)mbuf->buf_addr + 1698 mbuf->data_off); 1699 } /* there's no else as we take advantage of memset zeroing */ 1700 1701 /* Set TX offloads flags, if applicable */ 1702 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); 1703 1704 if (unlikely(mbuf->ol_flags & 1705 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 1706 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 1707 1708 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 1709 1710 /* Process first segment taking into 1711 * consideration pushed header 1712 */ 1713 if (mbuf->data_len > ena_tx_ctx.header_len) { 1714 ebuf->paddr = mbuf->buf_physaddr + 1715 mbuf->data_off + 1716 ena_tx_ctx.header_len; 1717 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 1718 ebuf++; 1719 tx_info->num_of_bufs++; 1720 } 1721 1722 while ((mbuf = mbuf->next) != NULL) { 1723 ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; 1724 ebuf->len = mbuf->data_len; 1725 ebuf++; 1726 tx_info->num_of_bufs++; 1727 } 1728 1729 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1730 1731 /* Write data to device */ 1732 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 1733 &ena_tx_ctx, &nb_hw_desc); 1734 if (unlikely(rc)) 1735 break; 1736 1737 tx_info->tx_descs = nb_hw_desc; 1738 1739 next_to_use++; 1740 } 1741 1742 /* If there are ready packets to be xmitted... */ 1743 if (sent_idx > 0) { 1744 /* ...let HW do its best :-) */ 1745 rte_wmb(); 1746 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 1747 1748 tx_ring->next_to_use = next_to_use; 1749 } 1750 1751 /* Clear complete packets */ 1752 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 1753 /* Get Tx info & store how many descs were processed */ 1754 tx_info = &tx_ring->tx_buffer_info[req_id]; 1755 total_tx_descs += tx_info->tx_descs; 1756 1757 /* Free whole mbuf chain */ 1758 mbuf = tx_info->mbuf; 1759 rte_pktmbuf_free(mbuf); 1760 1761 /* Put back descriptor to the ring for reuse */ 1762 tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; 1763 next_to_clean++; 1764 1765 /* If too many descs to clean, leave it for another run */ 1766 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 1767 break; 1768 } 1769 1770 if (total_tx_descs > 0) { 1771 /* acknowledge completion of sent packets */ 1772 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 1773 tx_ring->next_to_clean = next_to_clean; 1774 } 1775 1776 return sent_idx; 1777 } 1778 1779 static struct eth_driver rte_ena_pmd = { 1780 .pci_drv = { 1781 .id_table = pci_id_ena_map, 1782 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1783 .probe = rte_eth_dev_pci_probe, 1784 .remove = rte_eth_dev_pci_remove, 1785 }, 1786 .eth_dev_init = eth_ena_dev_init, 1787 .dev_private_size = sizeof(struct ena_adapter), 1788 }; 1789 1790 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd.pci_drv); 1791 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 1792 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio"); 1793