1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev.h> 36 #include <rte_tcp.h> 37 #include <rte_atomic.h> 38 #include <rte_dev.h> 39 #include <rte_errno.h> 40 #include <rte_version.h> 41 #include <rte_eal_memconfig.h> 42 43 #include "ena_ethdev.h" 44 #include "ena_logs.h" 45 #include "ena_platform.h" 46 #include "ena_com.h" 47 #include "ena_eth_com.h" 48 49 #include <ena_common_defs.h> 50 #include <ena_regs_defs.h> 51 #include <ena_admin_defs.h> 52 #include <ena_eth_io_defs.h> 53 54 #define DRV_MODULE_VER_MAJOR 1 55 #define DRV_MODULE_VER_MINOR 0 56 #define DRV_MODULE_VER_SUBMINOR 0 57 58 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 59 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 60 /*reverse version of ENA_IO_RXQ_IDX*/ 61 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 62 63 /* While processing submitted and completed descriptors (rx and tx path 64 * respectively) in a loop it is desired to: 65 * - perform batch submissions while populating sumbissmion queue 66 * - avoid blocking transmission of other packets during cleanup phase 67 * Hence the utilization ratio of 1/8 of a queue size. 68 */ 69 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 70 71 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 72 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 73 74 #define GET_L4_HDR_LEN(mbuf) \ 75 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 76 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 77 78 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 79 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 80 #define ENA_HASH_KEY_SIZE 40 81 #define ENA_ETH_SS_STATS 0xFF 82 #define ETH_GSTRING_LEN 32 83 84 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 85 86 enum ethtool_stringset { 87 ETH_SS_TEST = 0, 88 ETH_SS_STATS, 89 }; 90 91 struct ena_stats { 92 char name[ETH_GSTRING_LEN]; 93 int stat_offset; 94 }; 95 96 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 97 .name = #stat, \ 98 .stat_offset = offsetof(struct ena_com_stats_admin, stat) \ 99 } 100 101 #define ENA_STAT_ENTRY(stat, stat_type) { \ 102 .name = #stat, \ 103 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 104 } 105 106 #define ENA_STAT_RX_ENTRY(stat) \ 107 ENA_STAT_ENTRY(stat, rx) 108 109 #define ENA_STAT_TX_ENTRY(stat) \ 110 ENA_STAT_ENTRY(stat, tx) 111 112 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 113 ENA_STAT_ENTRY(stat, dev) 114 115 static const struct ena_stats ena_stats_global_strings[] = { 116 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 117 ENA_STAT_GLOBAL_ENTRY(io_suspend), 118 ENA_STAT_GLOBAL_ENTRY(io_resume), 119 ENA_STAT_GLOBAL_ENTRY(wd_expired), 120 ENA_STAT_GLOBAL_ENTRY(interface_up), 121 ENA_STAT_GLOBAL_ENTRY(interface_down), 122 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 123 }; 124 125 static const struct ena_stats ena_stats_tx_strings[] = { 126 ENA_STAT_TX_ENTRY(cnt), 127 ENA_STAT_TX_ENTRY(bytes), 128 ENA_STAT_TX_ENTRY(queue_stop), 129 ENA_STAT_TX_ENTRY(queue_wakeup), 130 ENA_STAT_TX_ENTRY(dma_mapping_err), 131 ENA_STAT_TX_ENTRY(linearize), 132 ENA_STAT_TX_ENTRY(linearize_failed), 133 ENA_STAT_TX_ENTRY(tx_poll), 134 ENA_STAT_TX_ENTRY(doorbells), 135 ENA_STAT_TX_ENTRY(prepare_ctx_err), 136 ENA_STAT_TX_ENTRY(missing_tx_comp), 137 ENA_STAT_TX_ENTRY(bad_req_id), 138 }; 139 140 static const struct ena_stats ena_stats_rx_strings[] = { 141 ENA_STAT_RX_ENTRY(cnt), 142 ENA_STAT_RX_ENTRY(bytes), 143 ENA_STAT_RX_ENTRY(refil_partial), 144 ENA_STAT_RX_ENTRY(bad_csum), 145 ENA_STAT_RX_ENTRY(page_alloc_fail), 146 ENA_STAT_RX_ENTRY(skb_alloc_fail), 147 ENA_STAT_RX_ENTRY(dma_mapping_err), 148 ENA_STAT_RX_ENTRY(bad_desc_num), 149 ENA_STAT_RX_ENTRY(small_copy_len_pkt), 150 }; 151 152 static const struct ena_stats ena_stats_ena_com_strings[] = { 153 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 154 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 155 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 156 ENA_STAT_ENA_COM_ENTRY(out_of_space), 157 ENA_STAT_ENA_COM_ENTRY(no_completion), 158 }; 159 160 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 161 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 162 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 163 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 164 165 /** Vendor ID used by Amazon devices */ 166 #define PCI_VENDOR_ID_AMAZON 0x1D0F 167 /** Amazon devices */ 168 #define PCI_DEVICE_ID_ENA_VF 0xEC20 169 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 170 171 static struct rte_pci_id pci_id_ena_map[] = { 172 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 173 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, 174 { .device_id = 0 }, 175 }; 176 177 static int ena_device_init(struct ena_com_dev *ena_dev, 178 struct ena_com_dev_get_features_ctx *get_feat_ctx); 179 static int ena_dev_configure(struct rte_eth_dev *dev); 180 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 181 uint16_t nb_pkts); 182 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 183 uint16_t nb_desc, unsigned int socket_id, 184 const struct rte_eth_txconf *tx_conf); 185 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 186 uint16_t nb_desc, unsigned int socket_id, 187 const struct rte_eth_rxconf *rx_conf, 188 struct rte_mempool *mp); 189 static uint16_t eth_ena_recv_pkts(void *rx_queue, 190 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 191 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 192 static void ena_init_rings(struct ena_adapter *adapter); 193 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 194 static int ena_start(struct rte_eth_dev *dev); 195 static void ena_close(struct rte_eth_dev *dev); 196 static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 197 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 198 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 199 static void ena_rx_queue_release(void *queue); 200 static void ena_tx_queue_release(void *queue); 201 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 202 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 203 static int ena_link_update(struct rte_eth_dev *dev, 204 __rte_unused int wait_to_complete); 205 static int ena_queue_restart(struct ena_ring *ring); 206 static int ena_queue_restart_all(struct rte_eth_dev *dev, 207 enum ena_ring_type ring_type); 208 static void ena_stats_restart(struct rte_eth_dev *dev); 209 static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, 210 struct rte_eth_dev_info *dev_info); 211 static int ena_rss_reta_update(struct rte_eth_dev *dev, 212 struct rte_eth_rss_reta_entry64 *reta_conf, 213 uint16_t reta_size); 214 static int ena_rss_reta_query(struct rte_eth_dev *dev, 215 struct rte_eth_rss_reta_entry64 *reta_conf, 216 uint16_t reta_size); 217 static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); 218 219 static struct eth_dev_ops ena_dev_ops = { 220 .dev_configure = ena_dev_configure, 221 .dev_infos_get = ena_infos_get, 222 .rx_queue_setup = ena_rx_queue_setup, 223 .tx_queue_setup = ena_tx_queue_setup, 224 .dev_start = ena_start, 225 .link_update = ena_link_update, 226 .stats_get = ena_stats_get, 227 .mtu_set = ena_mtu_set, 228 .rx_queue_release = ena_rx_queue_release, 229 .tx_queue_release = ena_tx_queue_release, 230 .dev_close = ena_close, 231 .reta_update = ena_rss_reta_update, 232 .reta_query = ena_rss_reta_query, 233 }; 234 235 #define NUMA_NO_NODE SOCKET_ID_ANY 236 237 static inline int ena_cpu_to_node(int cpu) 238 { 239 struct rte_config *config = rte_eal_get_configuration(); 240 241 if (likely(cpu < RTE_MAX_MEMZONE)) 242 return config->mem_config->memzone[cpu].socket_id; 243 244 return NUMA_NO_NODE; 245 } 246 247 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 248 struct ena_com_rx_ctx *ena_rx_ctx) 249 { 250 uint64_t ol_flags = 0; 251 252 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 253 ol_flags |= PKT_TX_TCP_CKSUM; 254 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 255 ol_flags |= PKT_TX_UDP_CKSUM; 256 257 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 258 ol_flags |= PKT_TX_IPV4; 259 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 260 ol_flags |= PKT_TX_IPV6; 261 262 if (unlikely(ena_rx_ctx->l4_csum_err)) 263 ol_flags |= PKT_RX_L4_CKSUM_BAD; 264 if (unlikely(ena_rx_ctx->l3_csum_err)) 265 ol_flags |= PKT_RX_IP_CKSUM_BAD; 266 267 mbuf->ol_flags = ol_flags; 268 } 269 270 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 271 struct ena_com_tx_ctx *ena_tx_ctx) 272 { 273 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 274 275 if (mbuf->ol_flags & 276 (PKT_TX_L4_MASK | PKT_TX_IP_CKSUM | PKT_TX_TCP_SEG)) { 277 /* check if TSO is required */ 278 if (mbuf->ol_flags & PKT_TX_TCP_SEG) { 279 ena_tx_ctx->tso_enable = true; 280 281 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 282 } 283 284 /* check if L3 checksum is needed */ 285 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 286 ena_tx_ctx->l3_csum_enable = true; 287 288 if (mbuf->ol_flags & PKT_TX_IPV6) { 289 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 290 } else { 291 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 292 293 /* set don't fragment (DF) flag */ 294 if (mbuf->packet_type & 295 (RTE_PTYPE_L4_NONFRAG 296 | RTE_PTYPE_INNER_L4_NONFRAG)) 297 ena_tx_ctx->df = true; 298 } 299 300 /* check if L4 checksum is needed */ 301 switch (mbuf->ol_flags & PKT_TX_L4_MASK) { 302 case PKT_TX_TCP_CKSUM: 303 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 304 ena_tx_ctx->l4_csum_enable = true; 305 break; 306 case PKT_TX_UDP_CKSUM: 307 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 308 ena_tx_ctx->l4_csum_enable = true; 309 break; 310 default: 311 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 312 ena_tx_ctx->l4_csum_enable = false; 313 break; 314 } 315 316 ena_meta->mss = mbuf->tso_segsz; 317 ena_meta->l3_hdr_len = mbuf->l3_len; 318 ena_meta->l3_hdr_offset = mbuf->l2_len; 319 /* this param needed only for TSO */ 320 ena_meta->l3_outer_hdr_len = 0; 321 ena_meta->l3_outer_hdr_offset = 0; 322 323 ena_tx_ctx->meta_valid = true; 324 } else { 325 ena_tx_ctx->meta_valid = false; 326 } 327 } 328 329 static void ena_config_host_info(struct ena_com_dev *ena_dev) 330 { 331 struct ena_admin_host_info *host_info; 332 int rc; 333 334 /* Allocate only the host info */ 335 rc = ena_com_allocate_host_info(ena_dev); 336 if (rc) { 337 RTE_LOG(ERR, PMD, "Cannot allocate host info\n"); 338 return; 339 } 340 341 host_info = ena_dev->host_attr.host_info; 342 343 host_info->os_type = ENA_ADMIN_OS_DPDK; 344 host_info->kernel_ver = RTE_VERSION; 345 snprintf((char *)host_info->kernel_ver_str, 346 sizeof(host_info->kernel_ver_str), 347 "%s", rte_version()); 348 host_info->os_dist = RTE_VERSION; 349 snprintf((char *)host_info->os_dist_str, 350 sizeof(host_info->os_dist_str), 351 "%s", rte_version()); 352 host_info->driver_version = 353 (DRV_MODULE_VER_MAJOR) | 354 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 355 (DRV_MODULE_VER_SUBMINOR << 356 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 357 358 rc = ena_com_set_host_attributes(ena_dev); 359 if (rc) { 360 if (rc == -EPERM) 361 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 362 else 363 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 364 365 goto err; 366 } 367 368 return; 369 370 err: 371 ena_com_delete_host_info(ena_dev); 372 } 373 374 static int 375 ena_get_sset_count(struct rte_eth_dev *dev, int sset) 376 { 377 if (sset != ETH_SS_STATS) 378 return -EOPNOTSUPP; 379 380 /* Workaround for clang: 381 * touch internal structures to prevent 382 * compiler error 383 */ 384 ENA_TOUCH(ena_stats_global_strings); 385 ENA_TOUCH(ena_stats_tx_strings); 386 ENA_TOUCH(ena_stats_rx_strings); 387 ENA_TOUCH(ena_stats_ena_com_strings); 388 389 return dev->data->nb_tx_queues * 390 (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) + 391 ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 392 } 393 394 static void ena_config_debug_area(struct ena_adapter *adapter) 395 { 396 u32 debug_area_size; 397 int rc, ss_count; 398 399 ss_count = ena_get_sset_count(adapter->rte_dev, ETH_SS_STATS); 400 if (ss_count <= 0) { 401 RTE_LOG(ERR, PMD, "SS count is negative\n"); 402 return; 403 } 404 405 /* allocate 32 bytes for each string and 64bit for the value */ 406 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 407 408 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 409 if (rc) { 410 RTE_LOG(ERR, PMD, "Cannot allocate debug area\n"); 411 return; 412 } 413 414 rc = ena_com_set_host_attributes(&adapter->ena_dev); 415 if (rc) { 416 if (rc == -EPERM) 417 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 418 else 419 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 420 goto err; 421 } 422 423 return; 424 err: 425 ena_com_delete_debug_area(&adapter->ena_dev); 426 } 427 428 static void ena_close(struct rte_eth_dev *dev) 429 { 430 struct ena_adapter *adapter = 431 (struct ena_adapter *)(dev->data->dev_private); 432 433 adapter->state = ENA_ADAPTER_STATE_STOPPED; 434 435 ena_rx_queue_release_all(dev); 436 ena_tx_queue_release_all(dev); 437 } 438 439 static int ena_rss_reta_update(struct rte_eth_dev *dev, 440 struct rte_eth_rss_reta_entry64 *reta_conf, 441 uint16_t reta_size) 442 { 443 struct ena_adapter *adapter = 444 (struct ena_adapter *)(dev->data->dev_private); 445 struct ena_com_dev *ena_dev = &adapter->ena_dev; 446 int ret, i; 447 u16 entry_value; 448 int conf_idx; 449 int idx; 450 451 if ((reta_size == 0) || (reta_conf == NULL)) 452 return -EINVAL; 453 454 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 455 RTE_LOG(WARNING, PMD, 456 "indirection table %d is bigger than supported (%d)\n", 457 reta_size, ENA_RX_RSS_TABLE_SIZE); 458 ret = -EINVAL; 459 goto err; 460 } 461 462 for (i = 0 ; i < reta_size ; i++) { 463 /* each reta_conf is for 64 entries. 464 * to support 128 we use 2 conf of 64 465 */ 466 conf_idx = i / RTE_RETA_GROUP_SIZE; 467 idx = i % RTE_RETA_GROUP_SIZE; 468 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 469 entry_value = 470 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 471 ret = ena_com_indirect_table_fill_entry(ena_dev, 472 i, 473 entry_value); 474 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 475 RTE_LOG(ERR, PMD, 476 "Cannot fill indirect table\n"); 477 ret = -ENOTSUP; 478 goto err; 479 } 480 } 481 } 482 483 ret = ena_com_indirect_table_set(ena_dev); 484 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 485 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 486 ret = -ENOTSUP; 487 goto err; 488 } 489 490 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 491 __func__, reta_size, adapter->rte_dev->data->port_id); 492 err: 493 return ret; 494 } 495 496 /* Query redirection table. */ 497 static int ena_rss_reta_query(struct rte_eth_dev *dev, 498 struct rte_eth_rss_reta_entry64 *reta_conf, 499 uint16_t reta_size) 500 { 501 struct ena_adapter *adapter = 502 (struct ena_adapter *)(dev->data->dev_private); 503 struct ena_com_dev *ena_dev = &adapter->ena_dev; 504 int ret; 505 int i; 506 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 507 int reta_conf_idx; 508 int reta_idx; 509 510 if (reta_size == 0 || reta_conf == NULL || 511 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 512 return -EINVAL; 513 514 ret = ena_com_indirect_table_get(ena_dev, indirect_table); 515 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 516 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 517 ret = -ENOTSUP; 518 goto err; 519 } 520 521 for (i = 0 ; i < reta_size ; i++) { 522 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 523 reta_idx = i % RTE_RETA_GROUP_SIZE; 524 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 525 reta_conf[reta_conf_idx].reta[reta_idx] = 526 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 527 } 528 err: 529 return ret; 530 } 531 532 static int ena_rss_init_default(struct ena_adapter *adapter) 533 { 534 struct ena_com_dev *ena_dev = &adapter->ena_dev; 535 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 536 int rc, i; 537 u32 val; 538 539 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 540 if (unlikely(rc)) { 541 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 542 goto err_rss_init; 543 } 544 545 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 546 val = i % nb_rx_queues; 547 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 548 ENA_IO_RXQ_IDX(val)); 549 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 550 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 551 goto err_fill_indir; 552 } 553 } 554 555 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 556 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 557 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 558 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 559 goto err_fill_indir; 560 } 561 562 rc = ena_com_set_default_hash_ctrl(ena_dev); 563 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 564 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 565 goto err_fill_indir; 566 } 567 568 rc = ena_com_indirect_table_set(ena_dev); 569 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 570 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 571 goto err_fill_indir; 572 } 573 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 574 adapter->rte_dev->data->port_id); 575 576 return 0; 577 578 err_fill_indir: 579 ena_com_rss_destroy(ena_dev); 580 err_rss_init: 581 582 return rc; 583 } 584 585 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 586 { 587 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 588 int nb_queues = dev->data->nb_rx_queues; 589 int i; 590 591 for (i = 0; i < nb_queues; i++) 592 ena_rx_queue_release(queues[i]); 593 } 594 595 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 596 { 597 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 598 int nb_queues = dev->data->nb_tx_queues; 599 int i; 600 601 for (i = 0; i < nb_queues; i++) 602 ena_tx_queue_release(queues[i]); 603 } 604 605 static void ena_rx_queue_release(void *queue) 606 { 607 struct ena_ring *ring = (struct ena_ring *)queue; 608 struct ena_adapter *adapter = ring->adapter; 609 int ena_qid; 610 611 ena_assert_msg(ring->configured, 612 "API violation - releasing not configured queue"); 613 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 614 "API violation"); 615 616 /* Destroy HW queue */ 617 ena_qid = ENA_IO_RXQ_IDX(ring->id); 618 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 619 620 /* Free all bufs */ 621 ena_rx_queue_release_bufs(ring); 622 623 /* Free ring resources */ 624 if (ring->rx_buffer_info) 625 rte_free(ring->rx_buffer_info); 626 ring->rx_buffer_info = NULL; 627 628 ring->configured = 0; 629 630 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 631 ring->port_id, ring->id); 632 } 633 634 static void ena_tx_queue_release(void *queue) 635 { 636 struct ena_ring *ring = (struct ena_ring *)queue; 637 struct ena_adapter *adapter = ring->adapter; 638 int ena_qid; 639 640 ena_assert_msg(ring->configured, 641 "API violation. Releasing not configured queue"); 642 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 643 "API violation"); 644 645 /* Destroy HW queue */ 646 ena_qid = ENA_IO_TXQ_IDX(ring->id); 647 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 648 649 /* Free all bufs */ 650 ena_tx_queue_release_bufs(ring); 651 652 /* Free ring resources */ 653 if (ring->tx_buffer_info) 654 rte_free(ring->tx_buffer_info); 655 656 if (ring->empty_tx_reqs) 657 rte_free(ring->empty_tx_reqs); 658 659 ring->empty_tx_reqs = NULL; 660 ring->tx_buffer_info = NULL; 661 662 ring->configured = 0; 663 664 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 665 ring->port_id, ring->id); 666 } 667 668 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 669 { 670 unsigned int ring_mask = ring->ring_size - 1; 671 672 while (ring->next_to_clean != ring->next_to_use) { 673 struct rte_mbuf *m = 674 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 675 676 if (m) 677 __rte_mbuf_raw_free(m); 678 679 ring->next_to_clean = 680 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 681 } 682 } 683 684 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 685 { 686 unsigned int ring_mask = ring->ring_size - 1; 687 688 while (ring->next_to_clean != ring->next_to_use) { 689 struct ena_tx_buffer *tx_buf = 690 &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; 691 692 if (tx_buf->mbuf) 693 rte_pktmbuf_free(tx_buf->mbuf); 694 695 ring->next_to_clean = 696 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 697 } 698 } 699 700 static int ena_link_update(struct rte_eth_dev *dev, 701 __rte_unused int wait_to_complete) 702 { 703 struct rte_eth_link *link = &dev->data->dev_link; 704 705 link->link_status = 1; 706 link->link_speed = ETH_SPEED_NUM_10G; 707 link->link_duplex = ETH_LINK_FULL_DUPLEX; 708 709 return 0; 710 } 711 712 static int ena_queue_restart_all(struct rte_eth_dev *dev, 713 enum ena_ring_type ring_type) 714 { 715 struct ena_adapter *adapter = 716 (struct ena_adapter *)(dev->data->dev_private); 717 struct ena_ring *queues = NULL; 718 int i = 0; 719 int rc = 0; 720 721 queues = (ring_type == ENA_RING_TYPE_RX) ? 722 adapter->rx_ring : adapter->tx_ring; 723 724 for (i = 0; i < adapter->num_queues; i++) { 725 if (queues[i].configured) { 726 if (ring_type == ENA_RING_TYPE_RX) { 727 ena_assert_msg( 728 dev->data->rx_queues[i] == &queues[i], 729 "Inconsistent state of rx queues\n"); 730 } else { 731 ena_assert_msg( 732 dev->data->tx_queues[i] == &queues[i], 733 "Inconsistent state of tx queues\n"); 734 } 735 736 rc = ena_queue_restart(&queues[i]); 737 738 if (rc) { 739 PMD_INIT_LOG(ERR, 740 "failed to restart queue %d type(%d)\n", 741 i, ring_type); 742 return -1; 743 } 744 } 745 } 746 747 return 0; 748 } 749 750 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 751 { 752 uint32_t max_frame_len = adapter->max_mtu; 753 754 if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) 755 max_frame_len = 756 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 757 758 return max_frame_len; 759 } 760 761 static int ena_check_valid_conf(struct ena_adapter *adapter) 762 { 763 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 764 765 if (max_frame_len > adapter->max_mtu) { 766 PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n", max_frame_len); 767 return -1; 768 } 769 770 return 0; 771 } 772 773 static int 774 ena_calc_queue_size(struct ena_com_dev *ena_dev, 775 struct ena_com_dev_get_features_ctx *get_feat_ctx) 776 { 777 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 778 779 queue_size = RTE_MIN(queue_size, 780 get_feat_ctx->max_queues.max_cq_depth); 781 queue_size = RTE_MIN(queue_size, 782 get_feat_ctx->max_queues.max_sq_depth); 783 784 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 785 queue_size = RTE_MIN(queue_size, 786 get_feat_ctx->max_queues.max_llq_depth); 787 788 /* Round down to power of 2 */ 789 if (!rte_is_power_of_2(queue_size)) 790 queue_size = rte_align32pow2(queue_size >> 1); 791 792 if (queue_size == 0) { 793 PMD_INIT_LOG(ERR, "Invalid queue size\n"); 794 return -EFAULT; 795 } 796 797 return queue_size; 798 } 799 800 static void ena_stats_restart(struct rte_eth_dev *dev) 801 { 802 struct ena_adapter *adapter = 803 (struct ena_adapter *)(dev->data->dev_private); 804 805 rte_atomic64_init(&adapter->drv_stats->ierrors); 806 rte_atomic64_init(&adapter->drv_stats->oerrors); 807 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 808 } 809 810 static void ena_stats_get(struct rte_eth_dev *dev, 811 struct rte_eth_stats *stats) 812 { 813 struct ena_admin_basic_stats ena_stats; 814 struct ena_adapter *adapter = 815 (struct ena_adapter *)(dev->data->dev_private); 816 struct ena_com_dev *ena_dev = &adapter->ena_dev; 817 int rc; 818 819 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 820 return; 821 822 memset(&ena_stats, 0, sizeof(ena_stats)); 823 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 824 if (unlikely(rc)) { 825 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 826 return; 827 } 828 829 /* Set of basic statistics from ENA */ 830 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 831 ena_stats.rx_pkts_low); 832 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 833 ena_stats.tx_pkts_low); 834 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 835 ena_stats.rx_bytes_low); 836 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 837 ena_stats.tx_bytes_low); 838 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 839 ena_stats.rx_drops_low); 840 841 /* Driver related stats */ 842 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 843 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 844 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 845 } 846 847 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 848 { 849 struct ena_adapter *adapter; 850 struct ena_com_dev *ena_dev; 851 int rc = 0; 852 853 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 854 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 855 adapter = (struct ena_adapter *)(dev->data->dev_private); 856 857 ena_dev = &adapter->ena_dev; 858 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 859 860 if (mtu > ena_get_mtu_conf(adapter)) { 861 RTE_LOG(ERR, PMD, 862 "Given MTU (%d) exceeds maximum MTU supported (%d)\n", 863 mtu, ena_get_mtu_conf(adapter)); 864 rc = -EINVAL; 865 goto err; 866 } 867 868 rc = ena_com_set_dev_mtu(ena_dev, mtu); 869 if (rc) 870 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 871 else 872 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 873 874 err: 875 return rc; 876 } 877 878 static int ena_start(struct rte_eth_dev *dev) 879 { 880 struct ena_adapter *adapter = 881 (struct ena_adapter *)(dev->data->dev_private); 882 int rc = 0; 883 884 if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || 885 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 886 PMD_INIT_LOG(ERR, "API violation"); 887 return -1; 888 } 889 890 rc = ena_check_valid_conf(adapter); 891 if (rc) 892 return rc; 893 894 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 895 if (rc) 896 return rc; 897 898 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 899 if (rc) 900 return rc; 901 902 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 903 ETH_MQ_RX_RSS_FLAG) { 904 rc = ena_rss_init_default(adapter); 905 if (rc) 906 return rc; 907 } 908 909 ena_stats_restart(dev); 910 911 adapter->state = ENA_ADAPTER_STATE_RUNNING; 912 913 return 0; 914 } 915 916 static int ena_queue_restart(struct ena_ring *ring) 917 { 918 int rc; 919 920 ena_assert_msg(ring->configured == 1, 921 "Trying to restart unconfigured queue\n"); 922 923 ring->next_to_clean = 0; 924 ring->next_to_use = 0; 925 926 if (ring->type == ENA_RING_TYPE_TX) 927 return 0; 928 929 rc = ena_populate_rx_queue(ring, ring->ring_size - 1); 930 if ((unsigned int)rc != ring->ring_size - 1) { 931 PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n"); 932 return (-1); 933 } 934 935 return 0; 936 } 937 938 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 939 uint16_t queue_idx, 940 uint16_t nb_desc, 941 __rte_unused unsigned int socket_id, 942 __rte_unused const struct rte_eth_txconf *tx_conf) 943 { 944 struct ena_com_create_io_ctx ctx = 945 /* policy set to _HOST just to satisfy icc compiler */ 946 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 947 ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 }; 948 struct ena_ring *txq = NULL; 949 struct ena_adapter *adapter = 950 (struct ena_adapter *)(dev->data->dev_private); 951 unsigned int i; 952 int ena_qid; 953 int rc; 954 struct ena_com_dev *ena_dev = &adapter->ena_dev; 955 956 txq = &adapter->tx_ring[queue_idx]; 957 958 if (txq->configured) { 959 RTE_LOG(CRIT, PMD, 960 "API violation. Queue %d is already configured\n", 961 queue_idx); 962 return -1; 963 } 964 965 if (nb_desc > adapter->tx_ring_size) { 966 RTE_LOG(ERR, PMD, 967 "Unsupported size of TX queue (max size: %d)\n", 968 adapter->tx_ring_size); 969 return -EINVAL; 970 } 971 972 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 973 974 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 975 ctx.qid = ena_qid; 976 ctx.msix_vector = -1; /* admin interrupts not used */ 977 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 978 ctx.queue_size = adapter->tx_ring_size; 979 ctx.numa_node = ena_cpu_to_node(queue_idx); 980 981 rc = ena_com_create_io_queue(ena_dev, &ctx); 982 if (rc) { 983 RTE_LOG(ERR, PMD, 984 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 985 queue_idx, ena_qid, rc); 986 } 987 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 988 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 989 990 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 991 &txq->ena_com_io_sq, 992 &txq->ena_com_io_cq); 993 if (rc) { 994 RTE_LOG(ERR, PMD, 995 "Failed to get TX queue handlers. TX queue num %d rc: %d\n", 996 queue_idx, rc); 997 ena_com_destroy_io_queue(ena_dev, ena_qid); 998 goto err; 999 } 1000 1001 txq->port_id = dev->data->port_id; 1002 txq->next_to_clean = 0; 1003 txq->next_to_use = 0; 1004 txq->ring_size = nb_desc; 1005 1006 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1007 sizeof(struct ena_tx_buffer) * 1008 txq->ring_size, 1009 RTE_CACHE_LINE_SIZE); 1010 if (!txq->tx_buffer_info) { 1011 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 1012 return -ENOMEM; 1013 } 1014 1015 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1016 sizeof(u16) * txq->ring_size, 1017 RTE_CACHE_LINE_SIZE); 1018 if (!txq->empty_tx_reqs) { 1019 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 1020 rte_free(txq->tx_buffer_info); 1021 return -ENOMEM; 1022 } 1023 for (i = 0; i < txq->ring_size; i++) 1024 txq->empty_tx_reqs[i] = i; 1025 1026 /* Store pointer to this queue in upper layer */ 1027 txq->configured = 1; 1028 dev->data->tx_queues[queue_idx] = txq; 1029 err: 1030 return rc; 1031 } 1032 1033 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1034 uint16_t queue_idx, 1035 uint16_t nb_desc, 1036 __rte_unused unsigned int socket_id, 1037 __rte_unused const struct rte_eth_rxconf *rx_conf, 1038 struct rte_mempool *mp) 1039 { 1040 struct ena_com_create_io_ctx ctx = 1041 /* policy set to _HOST just to satisfy icc compiler */ 1042 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1043 ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 }; 1044 struct ena_adapter *adapter = 1045 (struct ena_adapter *)(dev->data->dev_private); 1046 struct ena_ring *rxq = NULL; 1047 uint16_t ena_qid = 0; 1048 int rc = 0; 1049 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1050 1051 rxq = &adapter->rx_ring[queue_idx]; 1052 if (rxq->configured) { 1053 RTE_LOG(CRIT, PMD, 1054 "API violation. Queue %d is already configured\n", 1055 queue_idx); 1056 return -1; 1057 } 1058 1059 if (nb_desc > adapter->rx_ring_size) { 1060 RTE_LOG(ERR, PMD, 1061 "Unsupported size of RX queue (max size: %d)\n", 1062 adapter->rx_ring_size); 1063 return -EINVAL; 1064 } 1065 1066 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 1067 1068 ctx.qid = ena_qid; 1069 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1070 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1071 ctx.msix_vector = -1; /* admin interrupts not used */ 1072 ctx.queue_size = adapter->rx_ring_size; 1073 ctx.numa_node = ena_cpu_to_node(queue_idx); 1074 1075 rc = ena_com_create_io_queue(ena_dev, &ctx); 1076 if (rc) 1077 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 1078 queue_idx, rc); 1079 1080 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 1081 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1082 1083 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1084 &rxq->ena_com_io_sq, 1085 &rxq->ena_com_io_cq); 1086 if (rc) { 1087 RTE_LOG(ERR, PMD, 1088 "Failed to get RX queue handlers. RX queue num %d rc: %d\n", 1089 queue_idx, rc); 1090 ena_com_destroy_io_queue(ena_dev, ena_qid); 1091 } 1092 1093 rxq->port_id = dev->data->port_id; 1094 rxq->next_to_clean = 0; 1095 rxq->next_to_use = 0; 1096 rxq->ring_size = nb_desc; 1097 rxq->mb_pool = mp; 1098 1099 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1100 sizeof(struct rte_mbuf *) * nb_desc, 1101 RTE_CACHE_LINE_SIZE); 1102 if (!rxq->rx_buffer_info) { 1103 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 1104 return -ENOMEM; 1105 } 1106 1107 /* Store pointer to this queue in upper layer */ 1108 rxq->configured = 1; 1109 dev->data->rx_queues[queue_idx] = rxq; 1110 1111 return rc; 1112 } 1113 1114 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1115 { 1116 unsigned int i; 1117 int rc; 1118 unsigned int ring_size = rxq->ring_size; 1119 unsigned int ring_mask = ring_size - 1; 1120 int next_to_use = rxq->next_to_use & ring_mask; 1121 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 1122 1123 if (unlikely(!count)) 1124 return 0; 1125 1126 ena_assert_msg((((ENA_CIRC_COUNT(rxq->next_to_use, rxq->next_to_clean, 1127 rxq->ring_size)) + 1128 count) < rxq->ring_size), "bad ring state"); 1129 1130 count = RTE_MIN(count, ring_size - next_to_use); 1131 1132 /* get resources for incoming packets */ 1133 rc = rte_mempool_get_bulk(rxq->mb_pool, 1134 (void **)(&mbufs[next_to_use]), count); 1135 if (unlikely(rc < 0)) { 1136 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1137 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1138 return 0; 1139 } 1140 1141 for (i = 0; i < count; i++) { 1142 struct rte_mbuf *mbuf = mbufs[next_to_use]; 1143 struct ena_com_buf ebuf; 1144 1145 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 1146 /* prepare physical address for DMA transaction */ 1147 ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; 1148 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1149 /* pass resource to device */ 1150 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 1151 &ebuf, next_to_use); 1152 if (unlikely(rc)) { 1153 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 1154 break; 1155 } 1156 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, ring_size); 1157 } 1158 1159 /* When we submitted free recources to device... */ 1160 if (i > 0) { 1161 /* ...let HW know that it can fill buffers with data */ 1162 rte_wmb(); 1163 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1164 1165 rxq->next_to_use = next_to_use; 1166 } 1167 1168 return i; 1169 } 1170 1171 static int ena_device_init(struct ena_com_dev *ena_dev, 1172 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1173 { 1174 int rc; 1175 bool readless_supported; 1176 1177 /* Initialize mmio registers */ 1178 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1179 if (rc) { 1180 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 1181 return rc; 1182 } 1183 1184 /* The PCIe configuration space revision id indicate if mmio reg 1185 * read is disabled. 1186 */ 1187 readless_supported = 1188 !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id 1189 & ENA_MMIO_DISABLE_REG_READ); 1190 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1191 1192 /* reset device */ 1193 rc = ena_com_dev_reset(ena_dev); 1194 if (rc) { 1195 RTE_LOG(ERR, PMD, "cannot reset device\n"); 1196 goto err_mmio_read_less; 1197 } 1198 1199 /* check FW version */ 1200 rc = ena_com_validate_version(ena_dev); 1201 if (rc) { 1202 RTE_LOG(ERR, PMD, "device version is too low\n"); 1203 goto err_mmio_read_less; 1204 } 1205 1206 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1207 1208 /* ENA device administration layer init */ 1209 rc = ena_com_admin_init(ena_dev, NULL, true); 1210 if (rc) { 1211 RTE_LOG(ERR, PMD, 1212 "cannot initialize ena admin queue with device\n"); 1213 goto err_mmio_read_less; 1214 } 1215 1216 ena_config_host_info(ena_dev); 1217 1218 /* To enable the msix interrupts the driver needs to know the number 1219 * of queues. So the driver uses polling mode to retrieve this 1220 * information. 1221 */ 1222 ena_com_set_admin_polling_mode(ena_dev, true); 1223 1224 /* Get Device Attributes and features */ 1225 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1226 if (rc) { 1227 RTE_LOG(ERR, PMD, 1228 "cannot get attribute for ena device rc= %d\n", rc); 1229 goto err_admin_init; 1230 } 1231 1232 return 0; 1233 1234 err_admin_init: 1235 ena_com_admin_destroy(ena_dev); 1236 1237 err_mmio_read_less: 1238 ena_com_mmio_reg_read_request_destroy(ena_dev); 1239 1240 return rc; 1241 } 1242 1243 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1244 { 1245 struct rte_pci_device *pci_dev; 1246 struct ena_adapter *adapter = 1247 (struct ena_adapter *)(eth_dev->data->dev_private); 1248 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1249 struct ena_com_dev_get_features_ctx get_feat_ctx; 1250 int queue_size, rc; 1251 1252 static int adapters_found; 1253 1254 memset(adapter, 0, sizeof(struct ena_adapter)); 1255 ena_dev = &adapter->ena_dev; 1256 1257 eth_dev->dev_ops = &ena_dev_ops; 1258 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1259 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1260 adapter->rte_eth_dev_data = eth_dev->data; 1261 adapter->rte_dev = eth_dev; 1262 1263 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1264 return 0; 1265 1266 pci_dev = eth_dev->pci_dev; 1267 adapter->pdev = pci_dev; 1268 1269 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", 1270 pci_dev->addr.domain, 1271 pci_dev->addr.bus, 1272 pci_dev->addr.devid, 1273 pci_dev->addr.function); 1274 1275 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1276 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1277 1278 /* Present ENA_MEM_BAR indicates available LLQ mode. 1279 * Use corresponding policy 1280 */ 1281 if (adapter->dev_mem_base) 1282 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 1283 else if (adapter->regs) 1284 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1285 else 1286 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", 1287 ENA_REGS_BAR); 1288 1289 ena_dev->reg_bar = adapter->regs; 1290 ena_dev->dmadev = adapter->pdev; 1291 1292 adapter->id_number = adapters_found; 1293 1294 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1295 adapter->id_number); 1296 1297 /* device specific initialization routine */ 1298 rc = ena_device_init(ena_dev, &get_feat_ctx); 1299 if (rc) { 1300 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); 1301 return -1; 1302 } 1303 1304 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1305 if (get_feat_ctx.max_queues.max_llq_num == 0) { 1306 PMD_INIT_LOG(ERR, 1307 "Trying to use LLQ but llq_num is 0.\n" 1308 "Fall back into regular queues.\n"); 1309 ena_dev->tx_mem_queue_type = 1310 ENA_ADMIN_PLACEMENT_POLICY_HOST; 1311 adapter->num_queues = 1312 get_feat_ctx.max_queues.max_sq_num; 1313 } else { 1314 adapter->num_queues = 1315 get_feat_ctx.max_queues.max_llq_num; 1316 } 1317 } else { 1318 adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; 1319 } 1320 1321 queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); 1322 if ((queue_size <= 0) || (adapter->num_queues <= 0)) 1323 return -EFAULT; 1324 1325 adapter->tx_ring_size = queue_size; 1326 adapter->rx_ring_size = queue_size; 1327 1328 /* prepare ring structures */ 1329 ena_init_rings(adapter); 1330 1331 ena_config_debug_area(adapter); 1332 1333 /* Set max MTU for this device */ 1334 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1335 1336 /* Copy MAC address and point DPDK to it */ 1337 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1338 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1339 (struct ether_addr *)adapter->mac_addr); 1340 1341 adapter->drv_stats = rte_zmalloc("adapter stats", 1342 sizeof(*adapter->drv_stats), 1343 RTE_CACHE_LINE_SIZE); 1344 if (!adapter->drv_stats) { 1345 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1346 return -ENOMEM; 1347 } 1348 1349 adapters_found++; 1350 adapter->state = ENA_ADAPTER_STATE_INIT; 1351 1352 return 0; 1353 } 1354 1355 static int ena_dev_configure(struct rte_eth_dev *dev) 1356 { 1357 struct ena_adapter *adapter = 1358 (struct ena_adapter *)(dev->data->dev_private); 1359 1360 if (!(adapter->state == ENA_ADAPTER_STATE_INIT || 1361 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 1362 PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n", 1363 adapter->state); 1364 return -1; 1365 } 1366 1367 switch (adapter->state) { 1368 case ENA_ADAPTER_STATE_INIT: 1369 case ENA_ADAPTER_STATE_STOPPED: 1370 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1371 break; 1372 case ENA_ADAPTER_STATE_CONFIG: 1373 RTE_LOG(WARNING, PMD, 1374 "Ivalid driver state while trying to configure device\n"); 1375 break; 1376 default: 1377 break; 1378 } 1379 1380 return 0; 1381 } 1382 1383 static void ena_init_rings(struct ena_adapter *adapter) 1384 { 1385 int i; 1386 1387 for (i = 0; i < adapter->num_queues; i++) { 1388 struct ena_ring *ring = &adapter->tx_ring[i]; 1389 1390 ring->configured = 0; 1391 ring->type = ENA_RING_TYPE_TX; 1392 ring->adapter = adapter; 1393 ring->id = i; 1394 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1395 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1396 } 1397 1398 for (i = 0; i < adapter->num_queues; i++) { 1399 struct ena_ring *ring = &adapter->rx_ring[i]; 1400 1401 ring->configured = 0; 1402 ring->type = ENA_RING_TYPE_RX; 1403 ring->adapter = adapter; 1404 ring->id = i; 1405 } 1406 } 1407 1408 static void ena_infos_get(struct rte_eth_dev *dev, 1409 struct rte_eth_dev_info *dev_info) 1410 { 1411 struct ena_adapter *adapter; 1412 struct ena_com_dev *ena_dev; 1413 struct ena_com_dev_get_features_ctx feat; 1414 uint32_t rx_feat = 0, tx_feat = 0; 1415 int rc = 0; 1416 1417 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1418 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1419 adapter = (struct ena_adapter *)(dev->data->dev_private); 1420 1421 ena_dev = &adapter->ena_dev; 1422 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1423 1424 dev_info->speed_capa = 1425 ETH_LINK_SPEED_1G | 1426 ETH_LINK_SPEED_2_5G | 1427 ETH_LINK_SPEED_5G | 1428 ETH_LINK_SPEED_10G | 1429 ETH_LINK_SPEED_25G | 1430 ETH_LINK_SPEED_40G | 1431 ETH_LINK_SPEED_50G | 1432 ETH_LINK_SPEED_100G; 1433 1434 /* Get supported features from HW */ 1435 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1436 if (unlikely(rc)) { 1437 RTE_LOG(ERR, PMD, 1438 "Cannot get attribute for ena device rc= %d\n", rc); 1439 return; 1440 } 1441 1442 /* Set Tx & Rx features available for device */ 1443 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1444 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1445 1446 if (feat.offload.tx & 1447 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1448 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1449 DEV_TX_OFFLOAD_UDP_CKSUM | 1450 DEV_TX_OFFLOAD_TCP_CKSUM; 1451 1452 if (feat.offload.tx & 1453 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1454 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1455 DEV_RX_OFFLOAD_UDP_CKSUM | 1456 DEV_RX_OFFLOAD_TCP_CKSUM; 1457 1458 /* Inform framework about available features */ 1459 dev_info->rx_offload_capa = rx_feat; 1460 dev_info->tx_offload_capa = tx_feat; 1461 1462 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1463 dev_info->max_rx_pktlen = adapter->max_mtu; 1464 dev_info->max_mac_addrs = 1; 1465 1466 dev_info->max_rx_queues = adapter->num_queues; 1467 dev_info->max_tx_queues = adapter->num_queues; 1468 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1469 } 1470 1471 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1472 uint16_t nb_pkts) 1473 { 1474 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1475 unsigned int ring_size = rx_ring->ring_size; 1476 unsigned int ring_mask = ring_size - 1; 1477 uint16_t next_to_clean = rx_ring->next_to_clean; 1478 int desc_in_use = 0; 1479 unsigned int recv_idx = 0; 1480 struct rte_mbuf *mbuf = NULL; 1481 struct rte_mbuf *mbuf_head = NULL; 1482 struct rte_mbuf *mbuf_prev = NULL; 1483 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1484 unsigned int completed; 1485 1486 struct ena_com_rx_ctx ena_rx_ctx; 1487 int rc = 0; 1488 1489 /* Check adapter state */ 1490 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1491 RTE_LOG(ALERT, PMD, 1492 "Trying to receive pkts while device is NOT running\n"); 1493 return 0; 1494 } 1495 1496 desc_in_use = ENA_CIRC_COUNT(rx_ring->next_to_use, 1497 next_to_clean, ring_size); 1498 if (unlikely(nb_pkts > desc_in_use)) 1499 nb_pkts = desc_in_use; 1500 1501 for (completed = 0; completed < nb_pkts; completed++) { 1502 int segments = 0; 1503 1504 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1505 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1506 ena_rx_ctx.descs = 0; 1507 /* receive packet context */ 1508 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1509 rx_ring->ena_com_io_sq, 1510 &ena_rx_ctx); 1511 if (unlikely(rc)) { 1512 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1513 return 0; 1514 } 1515 1516 if (unlikely(ena_rx_ctx.descs == 0)) 1517 break; 1518 1519 while (segments < ena_rx_ctx.descs) { 1520 mbuf = rx_buff_info[next_to_clean & ring_mask]; 1521 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1522 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1523 mbuf->refcnt = 1; 1524 mbuf->next = NULL; 1525 if (segments == 0) { 1526 mbuf->nb_segs = ena_rx_ctx.descs; 1527 mbuf->port = rx_ring->port_id; 1528 mbuf->pkt_len = 0; 1529 mbuf_head = mbuf; 1530 } else { 1531 /* for multi-segment pkts create mbuf chain */ 1532 mbuf_prev->next = mbuf; 1533 } 1534 mbuf_head->pkt_len += mbuf->data_len; 1535 1536 mbuf_prev = mbuf; 1537 segments++; 1538 next_to_clean = 1539 ENA_RX_RING_IDX_NEXT(next_to_clean, ring_size); 1540 } 1541 1542 /* fill mbuf attributes if any */ 1543 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1544 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1545 1546 /* pass to DPDK application head mbuf */ 1547 rx_pkts[recv_idx] = mbuf_head; 1548 recv_idx++; 1549 } 1550 1551 /* Burst refill to save doorbells, memory barriers, const interval */ 1552 if (ring_size - desc_in_use - 1 > ENA_RING_DESCS_RATIO(ring_size)) 1553 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use - 1); 1554 1555 rx_ring->next_to_clean = next_to_clean & ring_mask; 1556 1557 return recv_idx; 1558 } 1559 1560 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1561 uint16_t nb_pkts) 1562 { 1563 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1564 unsigned int next_to_use = tx_ring->next_to_use; 1565 struct rte_mbuf *mbuf; 1566 unsigned int ring_size = tx_ring->ring_size; 1567 unsigned int ring_mask = ring_size - 1; 1568 struct ena_com_tx_ctx ena_tx_ctx; 1569 struct ena_tx_buffer *tx_info; 1570 struct ena_com_buf *ebuf; 1571 uint16_t rc, req_id, total_tx_descs = 0; 1572 uint16_t sent_idx = 0; 1573 int nb_hw_desc; 1574 1575 /* Check adapter state */ 1576 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1577 RTE_LOG(ALERT, PMD, 1578 "Trying to xmit pkts while device is NOT running\n"); 1579 return 0; 1580 } 1581 1582 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 1583 mbuf = tx_pkts[sent_idx]; 1584 1585 req_id = tx_ring->empty_tx_reqs[next_to_use]; 1586 tx_info = &tx_ring->tx_buffer_info[req_id]; 1587 tx_info->mbuf = mbuf; 1588 tx_info->num_of_bufs = 0; 1589 ebuf = tx_info->bufs; 1590 1591 /* Prepare TX context */ 1592 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1593 memset(&ena_tx_ctx.ena_meta, 0x0, 1594 sizeof(struct ena_com_tx_meta)); 1595 ena_tx_ctx.ena_bufs = ebuf; 1596 ena_tx_ctx.req_id = req_id; 1597 if (tx_ring->tx_mem_queue_type == 1598 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1599 /* prepare the push buffer with 1600 * virtual address of the data 1601 */ 1602 ena_tx_ctx.header_len = 1603 RTE_MIN(mbuf->data_len, 1604 tx_ring->tx_max_header_size); 1605 ena_tx_ctx.push_header = 1606 (void *)((char *)mbuf->buf_addr + 1607 mbuf->data_off); 1608 } /* there's no else as we take advantage of memset zeroing */ 1609 1610 /* Set TX offloads flags, if applicable */ 1611 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); 1612 1613 if (unlikely(mbuf->ol_flags & 1614 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 1615 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 1616 1617 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 1618 1619 /* Process first segment taking into 1620 * consideration pushed header 1621 */ 1622 if (mbuf->data_len > ena_tx_ctx.header_len) { 1623 ebuf->paddr = mbuf->buf_physaddr + 1624 mbuf->data_off + 1625 ena_tx_ctx.header_len; 1626 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 1627 ebuf++; 1628 tx_info->num_of_bufs++; 1629 } 1630 1631 while ((mbuf = mbuf->next) != NULL) { 1632 ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; 1633 ebuf->len = mbuf->data_len; 1634 ebuf++; 1635 tx_info->num_of_bufs++; 1636 } 1637 1638 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1639 1640 /* Write data to device */ 1641 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 1642 &ena_tx_ctx, &nb_hw_desc); 1643 if (unlikely(rc)) 1644 break; 1645 1646 tx_info->tx_descs = nb_hw_desc; 1647 1648 next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, ring_size); 1649 } 1650 1651 /* If there are ready packets to be xmitted... */ 1652 if (sent_idx > 0) { 1653 /* ...let HW do its best :-) */ 1654 rte_wmb(); 1655 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 1656 1657 tx_ring->next_to_use = next_to_use; 1658 } 1659 1660 /* Clear complete packets */ 1661 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 1662 /* Get Tx info & store how many descs were processed */ 1663 tx_info = &tx_ring->tx_buffer_info[req_id]; 1664 total_tx_descs += tx_info->tx_descs; 1665 1666 /* Free whole mbuf chain */ 1667 mbuf = tx_info->mbuf; 1668 rte_pktmbuf_free(mbuf); 1669 1670 /* Put back descriptor to the ring for reuse */ 1671 tx_ring->empty_tx_reqs[tx_ring->next_to_clean] = req_id; 1672 tx_ring->next_to_clean = 1673 ENA_TX_RING_IDX_NEXT(tx_ring->next_to_clean, 1674 tx_ring->ring_size); 1675 1676 /* If too many descs to clean, leave it for another run */ 1677 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 1678 break; 1679 } 1680 1681 if (total_tx_descs > 0) { 1682 /* acknowledge completion of sent packets */ 1683 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 1684 } 1685 1686 return sent_idx; 1687 } 1688 1689 static struct eth_driver rte_ena_pmd = { 1690 .pci_drv = { 1691 .id_table = pci_id_ena_map, 1692 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1693 .probe = rte_eth_dev_pci_probe, 1694 .remove = rte_eth_dev_pci_remove, 1695 }, 1696 .eth_dev_init = eth_ena_dev_init, 1697 .dev_private_size = sizeof(struct ena_adapter), 1698 }; 1699 1700 DRIVER_REGISTER_PCI(net_ena, rte_ena_pmd.pci_drv); 1701 DRIVER_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 1702