1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev.h> 36 #include <rte_tcp.h> 37 #include <rte_atomic.h> 38 #include <rte_dev.h> 39 #include <rte_errno.h> 40 #include <rte_version.h> 41 #include <rte_eal_memconfig.h> 42 43 #include "ena_ethdev.h" 44 #include "ena_logs.h" 45 #include "ena_platform.h" 46 #include "ena_com.h" 47 #include "ena_eth_com.h" 48 49 #include <ena_common_defs.h> 50 #include <ena_regs_defs.h> 51 #include <ena_admin_defs.h> 52 #include <ena_eth_io_defs.h> 53 54 #define DRV_MODULE_VER_MAJOR 1 55 #define DRV_MODULE_VER_MINOR 0 56 #define DRV_MODULE_VER_SUBMINOR 0 57 58 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 59 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 60 /*reverse version of ENA_IO_RXQ_IDX*/ 61 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 62 63 /* While processing submitted and completed descriptors (rx and tx path 64 * respectively) in a loop it is desired to: 65 * - perform batch submissions while populating sumbissmion queue 66 * - avoid blocking transmission of other packets during cleanup phase 67 * Hence the utilization ratio of 1/8 of a queue size. 68 */ 69 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 70 71 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 72 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 73 74 #define GET_L4_HDR_LEN(mbuf) \ 75 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 76 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 77 78 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 79 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 80 #define ENA_HASH_KEY_SIZE 40 81 #define ENA_ETH_SS_STATS 0xFF 82 #define ETH_GSTRING_LEN 32 83 84 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) 85 86 enum ethtool_stringset { 87 ETH_SS_TEST = 0, 88 ETH_SS_STATS, 89 }; 90 91 struct ena_stats { 92 char name[ETH_GSTRING_LEN]; 93 int stat_offset; 94 }; 95 96 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 97 .name = #stat, \ 98 .stat_offset = offsetof(struct ena_com_stats_admin, stat) \ 99 } 100 101 #define ENA_STAT_ENTRY(stat, stat_type) { \ 102 .name = #stat, \ 103 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ 104 } 105 106 #define ENA_STAT_RX_ENTRY(stat) \ 107 ENA_STAT_ENTRY(stat, rx) 108 109 #define ENA_STAT_TX_ENTRY(stat) \ 110 ENA_STAT_ENTRY(stat, tx) 111 112 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 113 ENA_STAT_ENTRY(stat, dev) 114 115 static const struct ena_stats ena_stats_global_strings[] = { 116 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 117 ENA_STAT_GLOBAL_ENTRY(io_suspend), 118 ENA_STAT_GLOBAL_ENTRY(io_resume), 119 ENA_STAT_GLOBAL_ENTRY(wd_expired), 120 ENA_STAT_GLOBAL_ENTRY(interface_up), 121 ENA_STAT_GLOBAL_ENTRY(interface_down), 122 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 123 }; 124 125 static const struct ena_stats ena_stats_tx_strings[] = { 126 ENA_STAT_TX_ENTRY(cnt), 127 ENA_STAT_TX_ENTRY(bytes), 128 ENA_STAT_TX_ENTRY(queue_stop), 129 ENA_STAT_TX_ENTRY(queue_wakeup), 130 ENA_STAT_TX_ENTRY(dma_mapping_err), 131 ENA_STAT_TX_ENTRY(linearize), 132 ENA_STAT_TX_ENTRY(linearize_failed), 133 ENA_STAT_TX_ENTRY(tx_poll), 134 ENA_STAT_TX_ENTRY(doorbells), 135 ENA_STAT_TX_ENTRY(prepare_ctx_err), 136 ENA_STAT_TX_ENTRY(missing_tx_comp), 137 ENA_STAT_TX_ENTRY(bad_req_id), 138 }; 139 140 static const struct ena_stats ena_stats_rx_strings[] = { 141 ENA_STAT_RX_ENTRY(cnt), 142 ENA_STAT_RX_ENTRY(bytes), 143 ENA_STAT_RX_ENTRY(refil_partial), 144 ENA_STAT_RX_ENTRY(bad_csum), 145 ENA_STAT_RX_ENTRY(page_alloc_fail), 146 ENA_STAT_RX_ENTRY(skb_alloc_fail), 147 ENA_STAT_RX_ENTRY(dma_mapping_err), 148 ENA_STAT_RX_ENTRY(bad_desc_num), 149 ENA_STAT_RX_ENTRY(small_copy_len_pkt), 150 }; 151 152 static const struct ena_stats ena_stats_ena_com_strings[] = { 153 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 154 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 155 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 156 ENA_STAT_ENA_COM_ENTRY(out_of_space), 157 ENA_STAT_ENA_COM_ENTRY(no_completion), 158 }; 159 160 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 161 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 162 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 163 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 164 165 /** Vendor ID used by Amazon devices */ 166 #define PCI_VENDOR_ID_AMAZON 0x1D0F 167 /** Amazon devices */ 168 #define PCI_DEVICE_ID_ENA_VF 0xEC20 169 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 170 171 static struct rte_pci_id pci_id_ena_map[] = { 172 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, 173 { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, 174 { .device_id = 0 }, 175 }; 176 177 static int ena_device_init(struct ena_com_dev *ena_dev, 178 struct ena_com_dev_get_features_ctx *get_feat_ctx); 179 static int ena_dev_configure(struct rte_eth_dev *dev); 180 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 181 uint16_t nb_pkts); 182 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 183 uint16_t nb_desc, unsigned int socket_id, 184 const struct rte_eth_txconf *tx_conf); 185 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 186 uint16_t nb_desc, unsigned int socket_id, 187 const struct rte_eth_rxconf *rx_conf, 188 struct rte_mempool *mp); 189 static uint16_t eth_ena_recv_pkts(void *rx_queue, 190 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 191 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 192 static void ena_init_rings(struct ena_adapter *adapter); 193 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 194 static int ena_start(struct rte_eth_dev *dev); 195 static void ena_close(struct rte_eth_dev *dev); 196 static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 197 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 198 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 199 static void ena_rx_queue_release(void *queue); 200 static void ena_tx_queue_release(void *queue); 201 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 202 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 203 static int ena_link_update(struct rte_eth_dev *dev, 204 __rte_unused int wait_to_complete); 205 static int ena_queue_restart(struct ena_ring *ring); 206 static int ena_queue_restart_all(struct rte_eth_dev *dev, 207 enum ena_ring_type ring_type); 208 static void ena_stats_restart(struct rte_eth_dev *dev); 209 static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, 210 struct rte_eth_dev_info *dev_info); 211 static int ena_rss_reta_update(struct rte_eth_dev *dev, 212 struct rte_eth_rss_reta_entry64 *reta_conf, 213 uint16_t reta_size); 214 static int ena_rss_reta_query(struct rte_eth_dev *dev, 215 struct rte_eth_rss_reta_entry64 *reta_conf, 216 uint16_t reta_size); 217 static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); 218 219 static struct eth_dev_ops ena_dev_ops = { 220 .dev_configure = ena_dev_configure, 221 .dev_infos_get = ena_infos_get, 222 .rx_queue_setup = ena_rx_queue_setup, 223 .tx_queue_setup = ena_tx_queue_setup, 224 .dev_start = ena_start, 225 .link_update = ena_link_update, 226 .stats_get = ena_stats_get, 227 .mtu_set = ena_mtu_set, 228 .rx_queue_release = ena_rx_queue_release, 229 .tx_queue_release = ena_tx_queue_release, 230 .dev_close = ena_close, 231 .reta_update = ena_rss_reta_update, 232 .reta_query = ena_rss_reta_query, 233 }; 234 235 #define NUMA_NO_NODE SOCKET_ID_ANY 236 237 static inline int ena_cpu_to_node(int cpu) 238 { 239 struct rte_config *config = rte_eal_get_configuration(); 240 241 if (likely(cpu < RTE_MAX_MEMZONE)) 242 return config->mem_config->memzone[cpu].socket_id; 243 244 return NUMA_NO_NODE; 245 } 246 247 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 248 struct ena_com_rx_ctx *ena_rx_ctx) 249 { 250 uint64_t ol_flags = 0; 251 252 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 253 ol_flags |= PKT_TX_TCP_CKSUM; 254 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 255 ol_flags |= PKT_TX_UDP_CKSUM; 256 257 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 258 ol_flags |= PKT_TX_IPV4; 259 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 260 ol_flags |= PKT_TX_IPV6; 261 262 if (unlikely(ena_rx_ctx->l4_csum_err)) 263 ol_flags |= PKT_RX_L4_CKSUM_BAD; 264 if (unlikely(ena_rx_ctx->l3_csum_err)) 265 ol_flags |= PKT_RX_IP_CKSUM_BAD; 266 267 mbuf->ol_flags = ol_flags; 268 } 269 270 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 271 struct ena_com_tx_ctx *ena_tx_ctx) 272 { 273 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 274 275 if (mbuf->ol_flags & 276 (PKT_TX_L4_MASK | PKT_TX_IP_CKSUM | PKT_TX_TCP_SEG)) { 277 /* check if TSO is required */ 278 if (mbuf->ol_flags & PKT_TX_TCP_SEG) { 279 ena_tx_ctx->tso_enable = true; 280 281 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 282 } 283 284 /* check if L3 checksum is needed */ 285 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 286 ena_tx_ctx->l3_csum_enable = true; 287 288 if (mbuf->ol_flags & PKT_TX_IPV6) { 289 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 290 } else { 291 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 292 293 /* set don't fragment (DF) flag */ 294 if (mbuf->packet_type & 295 (RTE_PTYPE_L4_NONFRAG 296 | RTE_PTYPE_INNER_L4_NONFRAG)) 297 ena_tx_ctx->df = true; 298 } 299 300 /* check if L4 checksum is needed */ 301 switch (mbuf->ol_flags & PKT_TX_L4_MASK) { 302 case PKT_TX_TCP_CKSUM: 303 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 304 ena_tx_ctx->l4_csum_enable = true; 305 break; 306 case PKT_TX_UDP_CKSUM: 307 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 308 ena_tx_ctx->l4_csum_enable = true; 309 break; 310 default: 311 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 312 ena_tx_ctx->l4_csum_enable = false; 313 break; 314 } 315 316 ena_meta->mss = mbuf->tso_segsz; 317 ena_meta->l3_hdr_len = mbuf->l3_len; 318 ena_meta->l3_hdr_offset = mbuf->l2_len; 319 /* this param needed only for TSO */ 320 ena_meta->l3_outer_hdr_len = 0; 321 ena_meta->l3_outer_hdr_offset = 0; 322 323 ena_tx_ctx->meta_valid = true; 324 } else { 325 ena_tx_ctx->meta_valid = false; 326 } 327 } 328 329 static void ena_config_host_info(struct ena_com_dev *ena_dev) 330 { 331 struct ena_admin_host_info *host_info; 332 int rc; 333 334 /* Allocate only the host info */ 335 rc = ena_com_allocate_host_info(ena_dev); 336 if (rc) { 337 RTE_LOG(ERR, PMD, "Cannot allocate host info\n"); 338 return; 339 } 340 341 host_info = ena_dev->host_attr.host_info; 342 343 host_info->os_type = ENA_ADMIN_OS_DPDK; 344 host_info->kernel_ver = RTE_VERSION; 345 strncpy((char *)host_info->kernel_ver_str, rte_version(), 346 strlen(rte_version())); 347 host_info->os_dist = RTE_VERSION; 348 strncpy((char *)host_info->os_dist_str, rte_version(), 349 strlen(rte_version())); 350 host_info->driver_version = 351 (DRV_MODULE_VER_MAJOR) | 352 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 353 (DRV_MODULE_VER_SUBMINOR << 354 ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 355 356 rc = ena_com_set_host_attributes(ena_dev); 357 if (rc) { 358 if (rc == -EPERM) 359 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 360 else 361 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 362 363 goto err; 364 } 365 366 return; 367 368 err: 369 ena_com_delete_host_info(ena_dev); 370 } 371 372 static int 373 ena_get_sset_count(struct rte_eth_dev *dev, int sset) 374 { 375 if (sset != ETH_SS_STATS) 376 return -EOPNOTSUPP; 377 378 /* Workaround for clang: 379 * touch internal structures to prevent 380 * compiler error 381 */ 382 ENA_TOUCH(ena_stats_global_strings); 383 ENA_TOUCH(ena_stats_tx_strings); 384 ENA_TOUCH(ena_stats_rx_strings); 385 ENA_TOUCH(ena_stats_ena_com_strings); 386 387 return dev->data->nb_tx_queues * 388 (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) + 389 ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 390 } 391 392 static void ena_config_debug_area(struct ena_adapter *adapter) 393 { 394 u32 debug_area_size; 395 int rc, ss_count; 396 397 ss_count = ena_get_sset_count(adapter->rte_dev, ETH_SS_STATS); 398 if (ss_count <= 0) { 399 RTE_LOG(ERR, PMD, "SS count is negative\n"); 400 return; 401 } 402 403 /* allocate 32 bytes for each string and 64bit for the value */ 404 debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; 405 406 rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); 407 if (rc) { 408 RTE_LOG(ERR, PMD, "Cannot allocate debug area\n"); 409 return; 410 } 411 412 rc = ena_com_set_host_attributes(&adapter->ena_dev); 413 if (rc) { 414 if (rc == -EPERM) 415 RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); 416 else 417 RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); 418 goto err; 419 } 420 421 return; 422 err: 423 ena_com_delete_debug_area(&adapter->ena_dev); 424 } 425 426 static void ena_close(struct rte_eth_dev *dev) 427 { 428 struct ena_adapter *adapter = 429 (struct ena_adapter *)(dev->data->dev_private); 430 431 adapter->state = ENA_ADAPTER_STATE_STOPPED; 432 433 ena_rx_queue_release_all(dev); 434 ena_tx_queue_release_all(dev); 435 } 436 437 static int ena_rss_reta_update(struct rte_eth_dev *dev, 438 struct rte_eth_rss_reta_entry64 *reta_conf, 439 uint16_t reta_size) 440 { 441 struct ena_adapter *adapter = 442 (struct ena_adapter *)(dev->data->dev_private); 443 struct ena_com_dev *ena_dev = &adapter->ena_dev; 444 int ret, i; 445 u16 entry_value; 446 int conf_idx; 447 int idx; 448 449 if ((reta_size == 0) || (reta_conf == NULL)) 450 return -EINVAL; 451 452 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 453 RTE_LOG(WARNING, PMD, 454 "indirection table %d is bigger than supported (%d)\n", 455 reta_size, ENA_RX_RSS_TABLE_SIZE); 456 ret = -EINVAL; 457 goto err; 458 } 459 460 for (i = 0 ; i < reta_size ; i++) { 461 /* each reta_conf is for 64 entries. 462 * to support 128 we use 2 conf of 64 463 */ 464 conf_idx = i / RTE_RETA_GROUP_SIZE; 465 idx = i % RTE_RETA_GROUP_SIZE; 466 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 467 entry_value = 468 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 469 ret = ena_com_indirect_table_fill_entry(ena_dev, 470 i, 471 entry_value); 472 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 473 RTE_LOG(ERR, PMD, 474 "Cannot fill indirect table\n"); 475 ret = -ENOTSUP; 476 goto err; 477 } 478 } 479 } 480 481 ret = ena_com_indirect_table_set(ena_dev); 482 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 483 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 484 ret = -ENOTSUP; 485 goto err; 486 } 487 488 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 489 __func__, reta_size, adapter->rte_dev->data->port_id); 490 err: 491 return ret; 492 } 493 494 /* Query redirection table. */ 495 static int ena_rss_reta_query(struct rte_eth_dev *dev, 496 struct rte_eth_rss_reta_entry64 *reta_conf, 497 uint16_t reta_size) 498 { 499 struct ena_adapter *adapter = 500 (struct ena_adapter *)(dev->data->dev_private); 501 struct ena_com_dev *ena_dev = &adapter->ena_dev; 502 int ret; 503 int i; 504 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 505 int reta_conf_idx; 506 int reta_idx; 507 508 if (reta_size == 0 || reta_conf == NULL || 509 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 510 return -EINVAL; 511 512 ret = ena_com_indirect_table_get(ena_dev, indirect_table); 513 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 514 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 515 ret = -ENOTSUP; 516 goto err; 517 } 518 519 for (i = 0 ; i < reta_size ; i++) { 520 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 521 reta_idx = i % RTE_RETA_GROUP_SIZE; 522 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 523 reta_conf[reta_conf_idx].reta[reta_idx] = 524 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 525 } 526 err: 527 return ret; 528 } 529 530 static int ena_rss_init_default(struct ena_adapter *adapter) 531 { 532 struct ena_com_dev *ena_dev = &adapter->ena_dev; 533 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 534 int rc, i; 535 u32 val; 536 537 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 538 if (unlikely(rc)) { 539 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 540 goto err_rss_init; 541 } 542 543 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 544 val = i % nb_rx_queues; 545 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 546 ENA_IO_RXQ_IDX(val)); 547 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 548 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 549 goto err_fill_indir; 550 } 551 } 552 553 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 554 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 555 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 556 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 557 goto err_fill_indir; 558 } 559 560 rc = ena_com_set_default_hash_ctrl(ena_dev); 561 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 562 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 563 goto err_fill_indir; 564 } 565 566 rc = ena_com_indirect_table_set(ena_dev); 567 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 568 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 569 goto err_fill_indir; 570 } 571 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 572 adapter->rte_dev->data->port_id); 573 574 return 0; 575 576 err_fill_indir: 577 ena_com_rss_destroy(ena_dev); 578 err_rss_init: 579 580 return rc; 581 } 582 583 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 584 { 585 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 586 int nb_queues = dev->data->nb_rx_queues; 587 int i; 588 589 for (i = 0; i < nb_queues; i++) 590 ena_rx_queue_release(queues[i]); 591 } 592 593 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 594 { 595 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 596 int nb_queues = dev->data->nb_tx_queues; 597 int i; 598 599 for (i = 0; i < nb_queues; i++) 600 ena_tx_queue_release(queues[i]); 601 } 602 603 static void ena_rx_queue_release(void *queue) 604 { 605 struct ena_ring *ring = (struct ena_ring *)queue; 606 struct ena_adapter *adapter = ring->adapter; 607 int ena_qid; 608 609 ena_assert_msg(ring->configured, 610 "API violation - releasing not configured queue"); 611 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 612 "API violation"); 613 614 /* Destroy HW queue */ 615 ena_qid = ENA_IO_RXQ_IDX(ring->id); 616 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 617 618 /* Free all bufs */ 619 ena_rx_queue_release_bufs(ring); 620 621 /* Free ring resources */ 622 if (ring->rx_buffer_info) 623 rte_free(ring->rx_buffer_info); 624 ring->rx_buffer_info = NULL; 625 626 ring->configured = 0; 627 628 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 629 ring->port_id, ring->id); 630 } 631 632 static void ena_tx_queue_release(void *queue) 633 { 634 struct ena_ring *ring = (struct ena_ring *)queue; 635 struct ena_adapter *adapter = ring->adapter; 636 int ena_qid; 637 638 ena_assert_msg(ring->configured, 639 "API violation. Releasing not configured queue"); 640 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 641 "API violation"); 642 643 /* Destroy HW queue */ 644 ena_qid = ENA_IO_TXQ_IDX(ring->id); 645 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 646 647 /* Free all bufs */ 648 ena_tx_queue_release_bufs(ring); 649 650 /* Free ring resources */ 651 if (ring->tx_buffer_info) 652 rte_free(ring->tx_buffer_info); 653 654 if (ring->empty_tx_reqs) 655 rte_free(ring->empty_tx_reqs); 656 657 ring->empty_tx_reqs = NULL; 658 ring->tx_buffer_info = NULL; 659 660 ring->configured = 0; 661 662 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 663 ring->port_id, ring->id); 664 } 665 666 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 667 { 668 unsigned int ring_mask = ring->ring_size - 1; 669 670 while (ring->next_to_clean != ring->next_to_use) { 671 struct rte_mbuf *m = 672 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 673 674 if (m) 675 __rte_mbuf_raw_free(m); 676 677 ring->next_to_clean = 678 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 679 } 680 } 681 682 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 683 { 684 unsigned int ring_mask = ring->ring_size - 1; 685 686 while (ring->next_to_clean != ring->next_to_use) { 687 struct ena_tx_buffer *tx_buf = 688 &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; 689 690 if (tx_buf->mbuf) 691 rte_pktmbuf_free(tx_buf->mbuf); 692 693 ring->next_to_clean = 694 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 695 } 696 } 697 698 static int ena_link_update(struct rte_eth_dev *dev, 699 __rte_unused int wait_to_complete) 700 { 701 struct rte_eth_link *link = &dev->data->dev_link; 702 703 link->link_status = 1; 704 link->link_speed = ETH_SPEED_NUM_10G; 705 link->link_duplex = ETH_LINK_FULL_DUPLEX; 706 707 return 0; 708 } 709 710 static int ena_queue_restart_all(struct rte_eth_dev *dev, 711 enum ena_ring_type ring_type) 712 { 713 struct ena_adapter *adapter = 714 (struct ena_adapter *)(dev->data->dev_private); 715 struct ena_ring *queues = NULL; 716 int i = 0; 717 int rc = 0; 718 719 queues = (ring_type == ENA_RING_TYPE_RX) ? 720 adapter->rx_ring : adapter->tx_ring; 721 722 for (i = 0; i < adapter->num_queues; i++) { 723 if (queues[i].configured) { 724 if (ring_type == ENA_RING_TYPE_RX) { 725 ena_assert_msg( 726 dev->data->rx_queues[i] == &queues[i], 727 "Inconsistent state of rx queues\n"); 728 } else { 729 ena_assert_msg( 730 dev->data->tx_queues[i] == &queues[i], 731 "Inconsistent state of tx queues\n"); 732 } 733 734 rc = ena_queue_restart(&queues[i]); 735 736 if (rc) { 737 PMD_INIT_LOG(ERR, 738 "failed to restart queue %d type(%d)\n", 739 i, ring_type); 740 return -1; 741 } 742 } 743 } 744 745 return 0; 746 } 747 748 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 749 { 750 uint32_t max_frame_len = adapter->max_mtu; 751 752 if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) 753 max_frame_len = 754 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 755 756 return max_frame_len; 757 } 758 759 static int ena_check_valid_conf(struct ena_adapter *adapter) 760 { 761 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 762 763 if (max_frame_len > adapter->max_mtu) { 764 PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n", max_frame_len); 765 return -1; 766 } 767 768 return 0; 769 } 770 771 static int 772 ena_calc_queue_size(struct ena_com_dev *ena_dev, 773 struct ena_com_dev_get_features_ctx *get_feat_ctx) 774 { 775 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 776 777 queue_size = RTE_MIN(queue_size, 778 get_feat_ctx->max_queues.max_cq_depth); 779 queue_size = RTE_MIN(queue_size, 780 get_feat_ctx->max_queues.max_sq_depth); 781 782 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 783 queue_size = RTE_MIN(queue_size, 784 get_feat_ctx->max_queues.max_llq_depth); 785 786 /* Round down to power of 2 */ 787 if (!rte_is_power_of_2(queue_size)) 788 queue_size = rte_align32pow2(queue_size >> 1); 789 790 if (queue_size == 0) { 791 PMD_INIT_LOG(ERR, "Invalid queue size\n"); 792 return -EFAULT; 793 } 794 795 return queue_size; 796 } 797 798 static void ena_stats_restart(struct rte_eth_dev *dev) 799 { 800 struct ena_adapter *adapter = 801 (struct ena_adapter *)(dev->data->dev_private); 802 803 rte_atomic64_init(&adapter->drv_stats->ierrors); 804 rte_atomic64_init(&adapter->drv_stats->oerrors); 805 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 806 } 807 808 static void ena_stats_get(struct rte_eth_dev *dev, 809 struct rte_eth_stats *stats) 810 { 811 struct ena_admin_basic_stats ena_stats; 812 struct ena_adapter *adapter = 813 (struct ena_adapter *)(dev->data->dev_private); 814 struct ena_com_dev *ena_dev = &adapter->ena_dev; 815 int rc; 816 817 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 818 return; 819 820 memset(&ena_stats, 0, sizeof(ena_stats)); 821 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 822 if (unlikely(rc)) { 823 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 824 return; 825 } 826 827 /* Set of basic statistics from ENA */ 828 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 829 ena_stats.rx_pkts_low); 830 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 831 ena_stats.tx_pkts_low); 832 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 833 ena_stats.rx_bytes_low); 834 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 835 ena_stats.tx_bytes_low); 836 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 837 ena_stats.rx_drops_low); 838 839 /* Driver related stats */ 840 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 841 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 842 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 843 } 844 845 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 846 { 847 struct ena_adapter *adapter; 848 struct ena_com_dev *ena_dev; 849 int rc = 0; 850 851 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 852 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 853 adapter = (struct ena_adapter *)(dev->data->dev_private); 854 855 ena_dev = &adapter->ena_dev; 856 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 857 858 if (mtu > ena_get_mtu_conf(adapter)) { 859 RTE_LOG(ERR, PMD, 860 "Given MTU (%d) exceeds maximum MTU supported (%d)\n", 861 mtu, ena_get_mtu_conf(adapter)); 862 rc = -EINVAL; 863 goto err; 864 } 865 866 rc = ena_com_set_dev_mtu(ena_dev, mtu); 867 if (rc) 868 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 869 else 870 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 871 872 err: 873 return rc; 874 } 875 876 static int ena_start(struct rte_eth_dev *dev) 877 { 878 struct ena_adapter *adapter = 879 (struct ena_adapter *)(dev->data->dev_private); 880 int rc = 0; 881 882 if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || 883 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 884 PMD_INIT_LOG(ERR, "API violation"); 885 return -1; 886 } 887 888 rc = ena_check_valid_conf(adapter); 889 if (rc) 890 return rc; 891 892 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 893 if (rc) 894 return rc; 895 896 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 897 if (rc) 898 return rc; 899 900 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 901 ETH_MQ_RX_RSS_FLAG) { 902 rc = ena_rss_init_default(adapter); 903 if (rc) 904 return rc; 905 } 906 907 ena_stats_restart(dev); 908 909 adapter->state = ENA_ADAPTER_STATE_RUNNING; 910 911 return 0; 912 } 913 914 static int ena_queue_restart(struct ena_ring *ring) 915 { 916 int rc; 917 918 ena_assert_msg(ring->configured == 1, 919 "Trying to restart unconfigured queue\n"); 920 921 ring->next_to_clean = 0; 922 ring->next_to_use = 0; 923 924 if (ring->type == ENA_RING_TYPE_TX) 925 return 0; 926 927 rc = ena_populate_rx_queue(ring, ring->ring_size - 1); 928 if ((unsigned int)rc != ring->ring_size - 1) { 929 PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n"); 930 return (-1); 931 } 932 933 return 0; 934 } 935 936 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 937 uint16_t queue_idx, 938 uint16_t nb_desc, 939 __rte_unused unsigned int socket_id, 940 __rte_unused const struct rte_eth_txconf *tx_conf) 941 { 942 struct ena_com_create_io_ctx ctx = 943 /* policy set to _HOST just to satisfy icc compiler */ 944 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 945 ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 }; 946 struct ena_ring *txq = NULL; 947 struct ena_adapter *adapter = 948 (struct ena_adapter *)(dev->data->dev_private); 949 unsigned int i; 950 int ena_qid; 951 int rc; 952 struct ena_com_dev *ena_dev = &adapter->ena_dev; 953 954 txq = &adapter->tx_ring[queue_idx]; 955 956 if (txq->configured) { 957 RTE_LOG(CRIT, PMD, 958 "API violation. Queue %d is already configured\n", 959 queue_idx); 960 return -1; 961 } 962 963 if (nb_desc > adapter->tx_ring_size) { 964 RTE_LOG(ERR, PMD, 965 "Unsupported size of TX queue (max size: %d)\n", 966 adapter->tx_ring_size); 967 return -EINVAL; 968 } 969 970 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 971 972 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 973 ctx.qid = ena_qid; 974 ctx.msix_vector = -1; /* admin interrupts not used */ 975 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 976 ctx.queue_size = adapter->tx_ring_size; 977 ctx.numa_node = ena_cpu_to_node(queue_idx); 978 979 rc = ena_com_create_io_queue(ena_dev, &ctx); 980 if (rc) { 981 RTE_LOG(ERR, PMD, 982 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 983 queue_idx, ena_qid, rc); 984 } 985 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 986 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 987 988 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 989 &txq->ena_com_io_sq, 990 &txq->ena_com_io_cq); 991 if (rc) { 992 RTE_LOG(ERR, PMD, 993 "Failed to get TX queue handlers. TX queue num %d rc: %d\n", 994 queue_idx, rc); 995 ena_com_destroy_io_queue(ena_dev, ena_qid); 996 goto err; 997 } 998 999 txq->port_id = dev->data->port_id; 1000 txq->next_to_clean = 0; 1001 txq->next_to_use = 0; 1002 txq->ring_size = nb_desc; 1003 1004 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 1005 sizeof(struct ena_tx_buffer) * 1006 txq->ring_size, 1007 RTE_CACHE_LINE_SIZE); 1008 if (!txq->tx_buffer_info) { 1009 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 1010 return -ENOMEM; 1011 } 1012 1013 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 1014 sizeof(u16) * txq->ring_size, 1015 RTE_CACHE_LINE_SIZE); 1016 if (!txq->empty_tx_reqs) { 1017 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 1018 rte_free(txq->tx_buffer_info); 1019 return -ENOMEM; 1020 } 1021 for (i = 0; i < txq->ring_size; i++) 1022 txq->empty_tx_reqs[i] = i; 1023 1024 /* Store pointer to this queue in upper layer */ 1025 txq->configured = 1; 1026 dev->data->tx_queues[queue_idx] = txq; 1027 err: 1028 return rc; 1029 } 1030 1031 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 1032 uint16_t queue_idx, 1033 uint16_t nb_desc, 1034 __rte_unused unsigned int socket_id, 1035 __rte_unused const struct rte_eth_rxconf *rx_conf, 1036 struct rte_mempool *mp) 1037 { 1038 struct ena_com_create_io_ctx ctx = 1039 /* policy set to _HOST just to satisfy icc compiler */ 1040 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 1041 ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 }; 1042 struct ena_adapter *adapter = 1043 (struct ena_adapter *)(dev->data->dev_private); 1044 struct ena_ring *rxq = NULL; 1045 uint16_t ena_qid = 0; 1046 int rc = 0; 1047 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1048 1049 rxq = &adapter->rx_ring[queue_idx]; 1050 if (rxq->configured) { 1051 RTE_LOG(CRIT, PMD, 1052 "API violation. Queue %d is already configured\n", 1053 queue_idx); 1054 return -1; 1055 } 1056 1057 if (nb_desc > adapter->rx_ring_size) { 1058 RTE_LOG(ERR, PMD, 1059 "Unsupported size of RX queue (max size: %d)\n", 1060 adapter->rx_ring_size); 1061 return -EINVAL; 1062 } 1063 1064 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 1065 1066 ctx.qid = ena_qid; 1067 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1068 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1069 ctx.msix_vector = -1; /* admin interrupts not used */ 1070 ctx.queue_size = adapter->rx_ring_size; 1071 ctx.numa_node = ena_cpu_to_node(queue_idx); 1072 1073 rc = ena_com_create_io_queue(ena_dev, &ctx); 1074 if (rc) 1075 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 1076 queue_idx, rc); 1077 1078 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 1079 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 1080 1081 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1082 &rxq->ena_com_io_sq, 1083 &rxq->ena_com_io_cq); 1084 if (rc) { 1085 RTE_LOG(ERR, PMD, 1086 "Failed to get RX queue handlers. RX queue num %d rc: %d\n", 1087 queue_idx, rc); 1088 ena_com_destroy_io_queue(ena_dev, ena_qid); 1089 } 1090 1091 rxq->port_id = dev->data->port_id; 1092 rxq->next_to_clean = 0; 1093 rxq->next_to_use = 0; 1094 rxq->ring_size = nb_desc; 1095 rxq->mb_pool = mp; 1096 1097 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 1098 sizeof(struct rte_mbuf *) * nb_desc, 1099 RTE_CACHE_LINE_SIZE); 1100 if (!rxq->rx_buffer_info) { 1101 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 1102 return -ENOMEM; 1103 } 1104 1105 /* Store pointer to this queue in upper layer */ 1106 rxq->configured = 1; 1107 dev->data->rx_queues[queue_idx] = rxq; 1108 1109 return rc; 1110 } 1111 1112 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 1113 { 1114 unsigned int i; 1115 int rc; 1116 unsigned int ring_size = rxq->ring_size; 1117 unsigned int ring_mask = ring_size - 1; 1118 int next_to_use = rxq->next_to_use & ring_mask; 1119 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 1120 1121 if (unlikely(!count)) 1122 return 0; 1123 1124 ena_assert_msg((((ENA_CIRC_COUNT(rxq->next_to_use, rxq->next_to_clean, 1125 rxq->ring_size)) + 1126 count) < rxq->ring_size), "bad ring state"); 1127 1128 count = RTE_MIN(count, ring_size - next_to_use); 1129 1130 /* get resources for incoming packets */ 1131 rc = rte_mempool_get_bulk(rxq->mb_pool, 1132 (void **)(&mbufs[next_to_use]), count); 1133 if (unlikely(rc < 0)) { 1134 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 1135 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 1136 return 0; 1137 } 1138 1139 for (i = 0; i < count; i++) { 1140 struct rte_mbuf *mbuf = mbufs[next_to_use]; 1141 struct ena_com_buf ebuf; 1142 1143 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 1144 /* prepare physical address for DMA transaction */ 1145 ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; 1146 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 1147 /* pass resource to device */ 1148 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 1149 &ebuf, next_to_use); 1150 if (unlikely(rc)) { 1151 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 1152 break; 1153 } 1154 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, ring_size); 1155 } 1156 1157 /* When we submitted free recources to device... */ 1158 if (i > 0) { 1159 /* ...let HW know that it can fill buffers with data */ 1160 rte_wmb(); 1161 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 1162 1163 rxq->next_to_use = next_to_use; 1164 } 1165 1166 return i; 1167 } 1168 1169 static int ena_device_init(struct ena_com_dev *ena_dev, 1170 struct ena_com_dev_get_features_ctx *get_feat_ctx) 1171 { 1172 int rc; 1173 bool readless_supported; 1174 1175 /* Initialize mmio registers */ 1176 rc = ena_com_mmio_reg_read_request_init(ena_dev); 1177 if (rc) { 1178 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 1179 return rc; 1180 } 1181 1182 /* The PCIe configuration space revision id indicate if mmio reg 1183 * read is disabled. 1184 */ 1185 readless_supported = 1186 !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id 1187 & ENA_MMIO_DISABLE_REG_READ); 1188 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 1189 1190 /* reset device */ 1191 rc = ena_com_dev_reset(ena_dev); 1192 if (rc) { 1193 RTE_LOG(ERR, PMD, "cannot reset device\n"); 1194 goto err_mmio_read_less; 1195 } 1196 1197 /* check FW version */ 1198 rc = ena_com_validate_version(ena_dev); 1199 if (rc) { 1200 RTE_LOG(ERR, PMD, "device version is too low\n"); 1201 goto err_mmio_read_less; 1202 } 1203 1204 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 1205 1206 /* ENA device administration layer init */ 1207 rc = ena_com_admin_init(ena_dev, NULL, true); 1208 if (rc) { 1209 RTE_LOG(ERR, PMD, 1210 "cannot initialize ena admin queue with device\n"); 1211 goto err_mmio_read_less; 1212 } 1213 1214 ena_config_host_info(ena_dev); 1215 1216 /* To enable the msix interrupts the driver needs to know the number 1217 * of queues. So the driver uses polling mode to retrieve this 1218 * information. 1219 */ 1220 ena_com_set_admin_polling_mode(ena_dev, true); 1221 1222 /* Get Device Attributes and features */ 1223 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1224 if (rc) { 1225 RTE_LOG(ERR, PMD, 1226 "cannot get attribute for ena device rc= %d\n", rc); 1227 goto err_admin_init; 1228 } 1229 1230 return 0; 1231 1232 err_admin_init: 1233 ena_com_admin_destroy(ena_dev); 1234 1235 err_mmio_read_less: 1236 ena_com_mmio_reg_read_request_destroy(ena_dev); 1237 1238 return rc; 1239 } 1240 1241 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1242 { 1243 struct rte_pci_device *pci_dev; 1244 struct ena_adapter *adapter = 1245 (struct ena_adapter *)(eth_dev->data->dev_private); 1246 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1247 struct ena_com_dev_get_features_ctx get_feat_ctx; 1248 int queue_size, rc; 1249 1250 static int adapters_found; 1251 1252 memset(adapter, 0, sizeof(struct ena_adapter)); 1253 ena_dev = &adapter->ena_dev; 1254 1255 eth_dev->dev_ops = &ena_dev_ops; 1256 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1257 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1258 adapter->rte_eth_dev_data = eth_dev->data; 1259 adapter->rte_dev = eth_dev; 1260 1261 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1262 return 0; 1263 1264 pci_dev = eth_dev->pci_dev; 1265 adapter->pdev = pci_dev; 1266 1267 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", 1268 pci_dev->addr.domain, 1269 pci_dev->addr.bus, 1270 pci_dev->addr.devid, 1271 pci_dev->addr.function); 1272 1273 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1274 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1275 1276 /* Present ENA_MEM_BAR indicates available LLQ mode. 1277 * Use corresponding policy 1278 */ 1279 if (adapter->dev_mem_base) 1280 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 1281 else if (adapter->regs) 1282 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1283 else 1284 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", 1285 ENA_REGS_BAR); 1286 1287 ena_dev->reg_bar = adapter->regs; 1288 ena_dev->dmadev = adapter->pdev; 1289 1290 adapter->id_number = adapters_found; 1291 1292 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1293 adapter->id_number); 1294 1295 /* device specific initialization routine */ 1296 rc = ena_device_init(ena_dev, &get_feat_ctx); 1297 if (rc) { 1298 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); 1299 return -1; 1300 } 1301 1302 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1303 if (get_feat_ctx.max_queues.max_llq_num == 0) { 1304 PMD_INIT_LOG(ERR, 1305 "Trying to use LLQ but llq_num is 0.\n" 1306 "Fall back into regular queues.\n"); 1307 ena_dev->tx_mem_queue_type = 1308 ENA_ADMIN_PLACEMENT_POLICY_HOST; 1309 adapter->num_queues = 1310 get_feat_ctx.max_queues.max_sq_num; 1311 } else { 1312 adapter->num_queues = 1313 get_feat_ctx.max_queues.max_llq_num; 1314 } 1315 } else { 1316 adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; 1317 } 1318 1319 queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); 1320 if ((queue_size <= 0) || (adapter->num_queues <= 0)) 1321 return -EFAULT; 1322 1323 adapter->tx_ring_size = queue_size; 1324 adapter->rx_ring_size = queue_size; 1325 1326 /* prepare ring structures */ 1327 ena_init_rings(adapter); 1328 1329 ena_config_debug_area(adapter); 1330 1331 /* Set max MTU for this device */ 1332 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1333 1334 /* Copy MAC address and point DPDK to it */ 1335 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1336 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1337 (struct ether_addr *)adapter->mac_addr); 1338 1339 adapter->drv_stats = rte_zmalloc("adapter stats", 1340 sizeof(*adapter->drv_stats), 1341 RTE_CACHE_LINE_SIZE); 1342 if (!adapter->drv_stats) { 1343 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1344 return -ENOMEM; 1345 } 1346 1347 adapters_found++; 1348 adapter->state = ENA_ADAPTER_STATE_INIT; 1349 1350 return 0; 1351 } 1352 1353 static int ena_dev_configure(struct rte_eth_dev *dev) 1354 { 1355 struct ena_adapter *adapter = 1356 (struct ena_adapter *)(dev->data->dev_private); 1357 1358 if (!(adapter->state == ENA_ADAPTER_STATE_INIT || 1359 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 1360 PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n", 1361 adapter->state); 1362 return -1; 1363 } 1364 1365 switch (adapter->state) { 1366 case ENA_ADAPTER_STATE_INIT: 1367 case ENA_ADAPTER_STATE_STOPPED: 1368 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1369 break; 1370 case ENA_ADAPTER_STATE_CONFIG: 1371 RTE_LOG(WARNING, PMD, 1372 "Ivalid driver state while trying to configure device\n"); 1373 break; 1374 default: 1375 break; 1376 } 1377 1378 return 0; 1379 } 1380 1381 static void ena_init_rings(struct ena_adapter *adapter) 1382 { 1383 int i; 1384 1385 for (i = 0; i < adapter->num_queues; i++) { 1386 struct ena_ring *ring = &adapter->tx_ring[i]; 1387 1388 ring->configured = 0; 1389 ring->type = ENA_RING_TYPE_TX; 1390 ring->adapter = adapter; 1391 ring->id = i; 1392 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1393 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1394 } 1395 1396 for (i = 0; i < adapter->num_queues; i++) { 1397 struct ena_ring *ring = &adapter->rx_ring[i]; 1398 1399 ring->configured = 0; 1400 ring->type = ENA_RING_TYPE_RX; 1401 ring->adapter = adapter; 1402 ring->id = i; 1403 } 1404 } 1405 1406 static void ena_infos_get(struct rte_eth_dev *dev, 1407 struct rte_eth_dev_info *dev_info) 1408 { 1409 struct ena_adapter *adapter; 1410 struct ena_com_dev *ena_dev; 1411 struct ena_com_dev_get_features_ctx feat; 1412 uint32_t rx_feat = 0, tx_feat = 0; 1413 int rc = 0; 1414 1415 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1416 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1417 adapter = (struct ena_adapter *)(dev->data->dev_private); 1418 1419 ena_dev = &adapter->ena_dev; 1420 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1421 1422 dev_info->speed_capa = 1423 ETH_LINK_SPEED_1G | 1424 ETH_LINK_SPEED_2_5G | 1425 ETH_LINK_SPEED_5G | 1426 ETH_LINK_SPEED_10G | 1427 ETH_LINK_SPEED_25G | 1428 ETH_LINK_SPEED_40G | 1429 ETH_LINK_SPEED_50G | 1430 ETH_LINK_SPEED_100G; 1431 1432 /* Get supported features from HW */ 1433 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1434 if (unlikely(rc)) { 1435 RTE_LOG(ERR, PMD, 1436 "Cannot get attribute for ena device rc= %d\n", rc); 1437 return; 1438 } 1439 1440 /* Set Tx & Rx features available for device */ 1441 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1442 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1443 1444 if (feat.offload.tx & 1445 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1446 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1447 DEV_TX_OFFLOAD_UDP_CKSUM | 1448 DEV_TX_OFFLOAD_TCP_CKSUM; 1449 1450 if (feat.offload.tx & 1451 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1452 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1453 DEV_RX_OFFLOAD_UDP_CKSUM | 1454 DEV_RX_OFFLOAD_TCP_CKSUM; 1455 1456 /* Inform framework about available features */ 1457 dev_info->rx_offload_capa = rx_feat; 1458 dev_info->tx_offload_capa = tx_feat; 1459 1460 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1461 dev_info->max_rx_pktlen = adapter->max_mtu; 1462 dev_info->max_mac_addrs = 1; 1463 1464 dev_info->max_rx_queues = adapter->num_queues; 1465 dev_info->max_tx_queues = adapter->num_queues; 1466 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1467 } 1468 1469 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1470 uint16_t nb_pkts) 1471 { 1472 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1473 unsigned int ring_size = rx_ring->ring_size; 1474 unsigned int ring_mask = ring_size - 1; 1475 uint16_t next_to_clean = rx_ring->next_to_clean; 1476 int desc_in_use = 0; 1477 unsigned int recv_idx = 0; 1478 struct rte_mbuf *mbuf = NULL; 1479 struct rte_mbuf *mbuf_head = NULL; 1480 struct rte_mbuf *mbuf_prev = NULL; 1481 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1482 unsigned int completed; 1483 1484 struct ena_com_rx_ctx ena_rx_ctx; 1485 int rc = 0; 1486 1487 /* Check adapter state */ 1488 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1489 RTE_LOG(ALERT, PMD, 1490 "Trying to receive pkts while device is NOT running\n"); 1491 return 0; 1492 } 1493 1494 desc_in_use = ENA_CIRC_COUNT(rx_ring->next_to_use, 1495 next_to_clean, ring_size); 1496 if (unlikely(nb_pkts > desc_in_use)) 1497 nb_pkts = desc_in_use; 1498 1499 for (completed = 0; completed < nb_pkts; completed++) { 1500 int segments = 0; 1501 1502 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1503 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1504 ena_rx_ctx.descs = 0; 1505 /* receive packet context */ 1506 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1507 rx_ring->ena_com_io_sq, 1508 &ena_rx_ctx); 1509 if (unlikely(rc)) { 1510 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1511 return 0; 1512 } 1513 1514 if (unlikely(ena_rx_ctx.descs == 0)) 1515 break; 1516 1517 while (segments < ena_rx_ctx.descs) { 1518 mbuf = rx_buff_info[next_to_clean & ring_mask]; 1519 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1520 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1521 mbuf->refcnt = 1; 1522 mbuf->next = NULL; 1523 if (segments == 0) { 1524 mbuf->nb_segs = ena_rx_ctx.descs; 1525 mbuf->port = rx_ring->port_id; 1526 mbuf->pkt_len = 0; 1527 mbuf_head = mbuf; 1528 } else { 1529 /* for multi-segment pkts create mbuf chain */ 1530 mbuf_prev->next = mbuf; 1531 } 1532 mbuf_head->pkt_len += mbuf->data_len; 1533 1534 mbuf_prev = mbuf; 1535 segments++; 1536 next_to_clean = 1537 ENA_RX_RING_IDX_NEXT(next_to_clean, ring_size); 1538 } 1539 1540 /* fill mbuf attributes if any */ 1541 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1542 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1543 1544 /* pass to DPDK application head mbuf */ 1545 rx_pkts[recv_idx] = mbuf_head; 1546 recv_idx++; 1547 } 1548 1549 /* Burst refill to save doorbells, memory barriers, const interval */ 1550 if (ring_size - desc_in_use - 1 > ENA_RING_DESCS_RATIO(ring_size)) 1551 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use - 1); 1552 1553 rx_ring->next_to_clean = next_to_clean & ring_mask; 1554 1555 return recv_idx; 1556 } 1557 1558 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1559 uint16_t nb_pkts) 1560 { 1561 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1562 unsigned int next_to_use = tx_ring->next_to_use; 1563 struct rte_mbuf *mbuf; 1564 unsigned int ring_size = tx_ring->ring_size; 1565 unsigned int ring_mask = ring_size - 1; 1566 struct ena_com_tx_ctx ena_tx_ctx; 1567 struct ena_tx_buffer *tx_info; 1568 struct ena_com_buf *ebuf; 1569 uint16_t rc, req_id, total_tx_descs = 0; 1570 uint16_t sent_idx = 0; 1571 int nb_hw_desc; 1572 1573 /* Check adapter state */ 1574 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1575 RTE_LOG(ALERT, PMD, 1576 "Trying to xmit pkts while device is NOT running\n"); 1577 return 0; 1578 } 1579 1580 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 1581 mbuf = tx_pkts[sent_idx]; 1582 1583 req_id = tx_ring->empty_tx_reqs[next_to_use]; 1584 tx_info = &tx_ring->tx_buffer_info[req_id]; 1585 tx_info->mbuf = mbuf; 1586 tx_info->num_of_bufs = 0; 1587 ebuf = tx_info->bufs; 1588 1589 /* Prepare TX context */ 1590 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1591 memset(&ena_tx_ctx.ena_meta, 0x0, 1592 sizeof(struct ena_com_tx_meta)); 1593 ena_tx_ctx.ena_bufs = ebuf; 1594 ena_tx_ctx.req_id = req_id; 1595 if (tx_ring->tx_mem_queue_type == 1596 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1597 /* prepare the push buffer with 1598 * virtual address of the data 1599 */ 1600 ena_tx_ctx.header_len = 1601 RTE_MIN(mbuf->data_len, 1602 tx_ring->tx_max_header_size); 1603 ena_tx_ctx.push_header = 1604 (void *)((char *)mbuf->buf_addr + 1605 mbuf->data_off); 1606 } /* there's no else as we take advantage of memset zeroing */ 1607 1608 /* Set TX offloads flags, if applicable */ 1609 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); 1610 1611 if (unlikely(mbuf->ol_flags & 1612 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 1613 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 1614 1615 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 1616 1617 /* Process first segment taking into 1618 * consideration pushed header 1619 */ 1620 if (mbuf->data_len > ena_tx_ctx.header_len) { 1621 ebuf->paddr = mbuf->buf_physaddr + 1622 mbuf->data_off + 1623 ena_tx_ctx.header_len; 1624 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 1625 ebuf++; 1626 tx_info->num_of_bufs++; 1627 } 1628 1629 while ((mbuf = mbuf->next) != NULL) { 1630 ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; 1631 ebuf->len = mbuf->data_len; 1632 ebuf++; 1633 tx_info->num_of_bufs++; 1634 } 1635 1636 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1637 1638 /* Write data to device */ 1639 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 1640 &ena_tx_ctx, &nb_hw_desc); 1641 if (unlikely(rc)) 1642 break; 1643 1644 tx_info->tx_descs = nb_hw_desc; 1645 1646 next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, ring_size); 1647 } 1648 1649 /* If there are ready packets to be xmitted... */ 1650 if (sent_idx > 0) { 1651 /* ...let HW do its best :-) */ 1652 rte_wmb(); 1653 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 1654 1655 tx_ring->next_to_use = next_to_use; 1656 } 1657 1658 /* Clear complete packets */ 1659 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 1660 /* Get Tx info & store how many descs were processed */ 1661 tx_info = &tx_ring->tx_buffer_info[req_id]; 1662 total_tx_descs += tx_info->tx_descs; 1663 1664 /* Free whole mbuf chain */ 1665 mbuf = tx_info->mbuf; 1666 rte_pktmbuf_free(mbuf); 1667 1668 /* Put back descriptor to the ring for reuse */ 1669 tx_ring->empty_tx_reqs[tx_ring->next_to_clean] = req_id; 1670 tx_ring->next_to_clean = 1671 ENA_TX_RING_IDX_NEXT(tx_ring->next_to_clean, 1672 tx_ring->ring_size); 1673 1674 /* If too many descs to clean, leave it for another run */ 1675 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 1676 break; 1677 } 1678 1679 if (total_tx_descs > 0) { 1680 /* acknowledge completion of sent packets */ 1681 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 1682 } 1683 1684 return sent_idx; 1685 } 1686 1687 static struct eth_driver rte_ena_pmd = { 1688 { 1689 .name = "rte_ena_pmd", 1690 .id_table = pci_id_ena_map, 1691 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1692 }, 1693 .eth_dev_init = eth_ena_dev_init, 1694 .dev_private_size = sizeof(struct ena_adapter), 1695 }; 1696 1697 static int 1698 rte_ena_pmd_init(const char *name __rte_unused, 1699 const char *params __rte_unused) 1700 { 1701 rte_eth_driver_register(&rte_ena_pmd); 1702 return 0; 1703 }; 1704 1705 struct rte_driver ena_pmd_drv = { 1706 .type = PMD_PDEV, 1707 .init = rte_ena_pmd_init, 1708 }; 1709 1710 PMD_REGISTER_DRIVER(ena_pmd_drv, net_ena); 1711 DRIVER_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); 1712