1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev.h> 36 #include <rte_tcp.h> 37 #include <rte_atomic.h> 38 #include <rte_dev.h> 39 #include <rte_errno.h> 40 41 #include "ena_ethdev.h" 42 #include "ena_logs.h" 43 #include "ena_platform.h" 44 #include "ena_com.h" 45 #include "ena_eth_com.h" 46 47 #include <ena_common_defs.h> 48 #include <ena_regs_defs.h> 49 #include <ena_admin_defs.h> 50 #include <ena_eth_io_defs.h> 51 52 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 53 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 54 /*reverse version of ENA_IO_RXQ_IDX*/ 55 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 56 57 /* While processing submitted and completed descriptors (rx and tx path 58 * respectively) in a loop it is desired to: 59 * - perform batch submissions while populating sumbissmion queue 60 * - avoid blocking transmission of other packets during cleanup phase 61 * Hence the utilization ratio of 1/8 of a queue size. 62 */ 63 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 64 65 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 66 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 67 68 #define GET_L4_HDR_LEN(mbuf) \ 69 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 70 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 71 72 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 73 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 74 #define ENA_HASH_KEY_SIZE 40 75 76 /** Vendor ID used by Amazon devices */ 77 #define PCI_VENDOR_ID_AMAZON 0x1D0F 78 /** Amazon devices */ 79 #define PCI_DEVICE_ID_ENA_VF 0xEC20 80 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 81 82 static struct rte_pci_id pci_id_ena_map[] = { 83 #define RTE_PCI_DEV_ID_DECL_ENA(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, 84 85 RTE_PCI_DEV_ID_DECL_ENA(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) 86 RTE_PCI_DEV_ID_DECL_ENA(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) 87 {.device_id = 0}, 88 }; 89 90 static int ena_device_init(struct ena_com_dev *ena_dev, 91 struct ena_com_dev_get_features_ctx *get_feat_ctx); 92 static int ena_dev_configure(struct rte_eth_dev *dev); 93 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 94 uint16_t nb_pkts); 95 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 96 uint16_t nb_desc, unsigned int socket_id, 97 const struct rte_eth_txconf *tx_conf); 98 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 99 uint16_t nb_desc, unsigned int socket_id, 100 const struct rte_eth_rxconf *rx_conf, 101 struct rte_mempool *mp); 102 static uint16_t eth_ena_recv_pkts(void *rx_queue, 103 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 104 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 105 static void ena_init_rings(struct ena_adapter *adapter); 106 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 107 static int ena_start(struct rte_eth_dev *dev); 108 static void ena_close(struct rte_eth_dev *dev); 109 static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 110 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 111 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 112 static void ena_rx_queue_release(void *queue); 113 static void ena_tx_queue_release(void *queue); 114 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 115 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 116 static int ena_link_update(struct rte_eth_dev *dev, 117 __rte_unused int wait_to_complete); 118 static int ena_queue_restart(struct ena_ring *ring); 119 static int ena_queue_restart_all(struct rte_eth_dev *dev, 120 enum ena_ring_type ring_type); 121 static void ena_stats_restart(struct rte_eth_dev *dev); 122 static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, 123 struct rte_eth_dev_info *dev_info); 124 static int ena_rss_reta_update(struct rte_eth_dev *dev, 125 struct rte_eth_rss_reta_entry64 *reta_conf, 126 uint16_t reta_size); 127 static int ena_rss_reta_query(struct rte_eth_dev *dev, 128 struct rte_eth_rss_reta_entry64 *reta_conf, 129 uint16_t reta_size); 130 131 static struct eth_dev_ops ena_dev_ops = { 132 .dev_configure = ena_dev_configure, 133 .dev_infos_get = ena_infos_get, 134 .rx_queue_setup = ena_rx_queue_setup, 135 .tx_queue_setup = ena_tx_queue_setup, 136 .dev_start = ena_start, 137 .link_update = ena_link_update, 138 .stats_get = ena_stats_get, 139 .mtu_set = ena_mtu_set, 140 .rx_queue_release = ena_rx_queue_release, 141 .tx_queue_release = ena_tx_queue_release, 142 .dev_close = ena_close, 143 .reta_update = ena_rss_reta_update, 144 .reta_query = ena_rss_reta_query, 145 }; 146 147 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 148 struct ena_com_rx_ctx *ena_rx_ctx) 149 { 150 uint64_t ol_flags = 0; 151 152 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 153 ol_flags |= PKT_TX_TCP_CKSUM; 154 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 155 ol_flags |= PKT_TX_UDP_CKSUM; 156 157 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 158 ol_flags |= PKT_TX_IPV4; 159 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 160 ol_flags |= PKT_TX_IPV6; 161 162 if (unlikely(ena_rx_ctx->l4_csum_err)) 163 ol_flags |= PKT_RX_L4_CKSUM_BAD; 164 if (unlikely(ena_rx_ctx->l3_csum_err)) 165 ol_flags |= PKT_RX_IP_CKSUM_BAD; 166 167 mbuf->ol_flags = ol_flags; 168 } 169 170 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 171 struct ena_com_tx_ctx *ena_tx_ctx) 172 { 173 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 174 175 if (mbuf->ol_flags & 176 (PKT_TX_L4_MASK | PKT_TX_IP_CKSUM | PKT_TX_TCP_SEG)) { 177 /* check if TSO is required */ 178 if (mbuf->ol_flags & PKT_TX_TCP_SEG) { 179 ena_tx_ctx->tso_enable = true; 180 181 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 182 } 183 184 /* check if L3 checksum is needed */ 185 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 186 ena_tx_ctx->l3_csum_enable = true; 187 188 if (mbuf->ol_flags & PKT_TX_IPV6) { 189 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 190 } else { 191 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 192 193 /* set don't fragment (DF) flag */ 194 if (mbuf->packet_type & 195 (RTE_PTYPE_L4_NONFRAG 196 | RTE_PTYPE_INNER_L4_NONFRAG)) 197 ena_tx_ctx->df = true; 198 } 199 200 /* check if L4 checksum is needed */ 201 switch (mbuf->ol_flags & PKT_TX_L4_MASK) { 202 case PKT_TX_TCP_CKSUM: 203 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 204 ena_tx_ctx->l4_csum_enable = true; 205 break; 206 case PKT_TX_UDP_CKSUM: 207 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 208 ena_tx_ctx->l4_csum_enable = true; 209 break; 210 default: 211 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 212 ena_tx_ctx->l4_csum_enable = false; 213 break; 214 } 215 216 ena_meta->mss = mbuf->tso_segsz; 217 ena_meta->l3_hdr_len = mbuf->l3_len; 218 ena_meta->l3_hdr_offset = mbuf->l2_len; 219 /* this param needed only for TSO */ 220 ena_meta->l3_outer_hdr_len = 0; 221 ena_meta->l3_outer_hdr_offset = 0; 222 223 ena_tx_ctx->meta_valid = true; 224 } else { 225 ena_tx_ctx->meta_valid = false; 226 } 227 } 228 229 static void ena_close(struct rte_eth_dev *dev) 230 { 231 struct ena_adapter *adapter = 232 (struct ena_adapter *)(dev->data->dev_private); 233 234 adapter->state = ENA_ADAPTER_STATE_STOPPED; 235 236 ena_rx_queue_release_all(dev); 237 ena_tx_queue_release_all(dev); 238 } 239 240 static int ena_rss_reta_update(struct rte_eth_dev *dev, 241 struct rte_eth_rss_reta_entry64 *reta_conf, 242 uint16_t reta_size) 243 { 244 struct ena_adapter *adapter = 245 (struct ena_adapter *)(dev->data->dev_private); 246 struct ena_com_dev *ena_dev = &adapter->ena_dev; 247 int ret, i; 248 u16 entry_value; 249 int conf_idx; 250 int idx; 251 252 if ((reta_size == 0) || (reta_conf == NULL)) 253 return -EINVAL; 254 255 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 256 RTE_LOG(WARNING, PMD, 257 "indirection table %d is bigger than supported (%d)\n", 258 reta_size, ENA_RX_RSS_TABLE_SIZE); 259 ret = -EINVAL; 260 goto err; 261 } 262 263 for (i = 0 ; i < reta_size ; i++) { 264 /* each reta_conf is for 64 entries. 265 * to support 128 we use 2 conf of 64 266 */ 267 conf_idx = i / RTE_RETA_GROUP_SIZE; 268 idx = i % RTE_RETA_GROUP_SIZE; 269 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 270 entry_value = 271 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 272 ret = ena_com_indirect_table_fill_entry(ena_dev, 273 i, 274 entry_value); 275 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 276 RTE_LOG(ERR, PMD, 277 "Cannot fill indirect table\n"); 278 ret = -ENOTSUP; 279 goto err; 280 } 281 } 282 } 283 284 ret = ena_com_indirect_table_set(ena_dev); 285 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 286 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 287 ret = -ENOTSUP; 288 goto err; 289 } 290 291 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 292 __func__, reta_size, adapter->rte_dev->data->port_id); 293 err: 294 return ret; 295 } 296 297 /* Query redirection table. */ 298 static int ena_rss_reta_query(struct rte_eth_dev *dev, 299 struct rte_eth_rss_reta_entry64 *reta_conf, 300 uint16_t reta_size) 301 { 302 struct ena_adapter *adapter = 303 (struct ena_adapter *)(dev->data->dev_private); 304 struct ena_com_dev *ena_dev = &adapter->ena_dev; 305 int ret; 306 int i; 307 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 308 int reta_conf_idx; 309 int reta_idx; 310 311 if (reta_size == 0 || reta_conf == NULL || 312 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 313 return -EINVAL; 314 315 ret = ena_com_indirect_table_get(ena_dev, indirect_table); 316 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 317 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 318 ret = -ENOTSUP; 319 goto err; 320 } 321 322 for (i = 0 ; i < reta_size ; i++) { 323 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 324 reta_idx = i % RTE_RETA_GROUP_SIZE; 325 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 326 reta_conf[reta_conf_idx].reta[reta_idx] = 327 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 328 } 329 err: 330 return ret; 331 } 332 333 static int ena_rss_init_default(struct ena_adapter *adapter) 334 { 335 struct ena_com_dev *ena_dev = &adapter->ena_dev; 336 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 337 int rc, i; 338 u32 val; 339 340 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 341 if (unlikely(rc)) { 342 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 343 goto err_rss_init; 344 } 345 346 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 347 val = i % nb_rx_queues; 348 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 349 ENA_IO_RXQ_IDX(val)); 350 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 351 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 352 goto err_fill_indir; 353 } 354 } 355 356 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 357 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 358 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 359 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 360 goto err_fill_indir; 361 } 362 363 rc = ena_com_set_default_hash_ctrl(ena_dev); 364 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 365 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 366 goto err_fill_indir; 367 } 368 369 rc = ena_com_indirect_table_set(ena_dev); 370 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 371 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 372 goto err_fill_indir; 373 } 374 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 375 adapter->rte_dev->data->port_id); 376 377 return 0; 378 379 err_fill_indir: 380 ena_com_rss_destroy(ena_dev); 381 err_rss_init: 382 383 return rc; 384 } 385 386 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 387 { 388 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 389 int nb_queues = dev->data->nb_rx_queues; 390 int i; 391 392 for (i = 0; i < nb_queues; i++) 393 ena_rx_queue_release(queues[i]); 394 } 395 396 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 397 { 398 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 399 int nb_queues = dev->data->nb_tx_queues; 400 int i; 401 402 for (i = 0; i < nb_queues; i++) 403 ena_tx_queue_release(queues[i]); 404 } 405 406 static void ena_rx_queue_release(void *queue) 407 { 408 struct ena_ring *ring = (struct ena_ring *)queue; 409 struct ena_adapter *adapter = ring->adapter; 410 int ena_qid; 411 412 ena_assert_msg(ring->configured, 413 "API violation - releasing not configured queue"); 414 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 415 "API violation"); 416 417 /* Destroy HW queue */ 418 ena_qid = ENA_IO_RXQ_IDX(ring->id); 419 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 420 421 /* Free all bufs */ 422 ena_rx_queue_release_bufs(ring); 423 424 /* Free ring resources */ 425 if (ring->rx_buffer_info) 426 rte_free(ring->rx_buffer_info); 427 ring->rx_buffer_info = NULL; 428 429 ring->configured = 0; 430 431 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 432 ring->port_id, ring->id); 433 } 434 435 static void ena_tx_queue_release(void *queue) 436 { 437 struct ena_ring *ring = (struct ena_ring *)queue; 438 struct ena_adapter *adapter = ring->adapter; 439 int ena_qid; 440 441 ena_assert_msg(ring->configured, 442 "API violation. Releasing not configured queue"); 443 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 444 "API violation"); 445 446 /* Destroy HW queue */ 447 ena_qid = ENA_IO_TXQ_IDX(ring->id); 448 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 449 450 /* Free all bufs */ 451 ena_tx_queue_release_bufs(ring); 452 453 /* Free ring resources */ 454 if (ring->tx_buffer_info) 455 rte_free(ring->tx_buffer_info); 456 457 if (ring->empty_tx_reqs) 458 rte_free(ring->empty_tx_reqs); 459 460 ring->empty_tx_reqs = NULL; 461 ring->tx_buffer_info = NULL; 462 463 ring->configured = 0; 464 465 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 466 ring->port_id, ring->id); 467 } 468 469 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 470 { 471 unsigned int ring_mask = ring->ring_size - 1; 472 473 while (ring->next_to_clean != ring->next_to_use) { 474 struct rte_mbuf *m = 475 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 476 477 if (m) 478 __rte_mbuf_raw_free(m); 479 480 ring->next_to_clean = 481 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 482 } 483 } 484 485 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 486 { 487 unsigned int ring_mask = ring->ring_size - 1; 488 489 while (ring->next_to_clean != ring->next_to_use) { 490 struct ena_tx_buffer *tx_buf = 491 &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; 492 493 if (tx_buf->mbuf) 494 rte_pktmbuf_free(tx_buf->mbuf); 495 496 ring->next_to_clean = 497 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 498 } 499 } 500 501 static int ena_link_update(struct rte_eth_dev *dev, 502 __rte_unused int wait_to_complete) 503 { 504 struct rte_eth_link *link = &dev->data->dev_link; 505 506 link->link_status = 1; 507 link->link_speed = ETH_SPEED_NUM_10G; 508 link->link_duplex = ETH_LINK_FULL_DUPLEX; 509 510 return 0; 511 } 512 513 static int ena_queue_restart_all(struct rte_eth_dev *dev, 514 enum ena_ring_type ring_type) 515 { 516 struct ena_adapter *adapter = 517 (struct ena_adapter *)(dev->data->dev_private); 518 struct ena_ring *queues = NULL; 519 int i = 0; 520 int rc = 0; 521 522 queues = (ring_type == ENA_RING_TYPE_RX) ? 523 adapter->rx_ring : adapter->tx_ring; 524 525 for (i = 0; i < adapter->num_queues; i++) { 526 if (queues[i].configured) { 527 if (ring_type == ENA_RING_TYPE_RX) { 528 ena_assert_msg( 529 dev->data->rx_queues[i] == &queues[i], 530 "Inconsistent state of rx queues\n"); 531 } else { 532 ena_assert_msg( 533 dev->data->tx_queues[i] == &queues[i], 534 "Inconsistent state of tx queues\n"); 535 } 536 537 rc = ena_queue_restart(&queues[i]); 538 539 if (rc) { 540 PMD_INIT_LOG(ERR, 541 "failed to restart queue %d type(%d)\n", 542 i, ring_type); 543 return -1; 544 } 545 } 546 } 547 548 return 0; 549 } 550 551 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 552 { 553 uint32_t max_frame_len = adapter->max_mtu; 554 555 if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) 556 max_frame_len = 557 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 558 559 return max_frame_len; 560 } 561 562 static int ena_check_valid_conf(struct ena_adapter *adapter) 563 { 564 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 565 566 if (max_frame_len > adapter->max_mtu) { 567 PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n", max_frame_len); 568 return -1; 569 } 570 571 return 0; 572 } 573 574 static int 575 ena_calc_queue_size(struct ena_com_dev *ena_dev, 576 struct ena_com_dev_get_features_ctx *get_feat_ctx) 577 { 578 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 579 580 queue_size = RTE_MIN(queue_size, 581 get_feat_ctx->max_queues.max_cq_depth); 582 queue_size = RTE_MIN(queue_size, 583 get_feat_ctx->max_queues.max_sq_depth); 584 585 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 586 queue_size = RTE_MIN(queue_size, 587 get_feat_ctx->max_queues.max_llq_depth); 588 589 /* Round down to power of 2 */ 590 if (!rte_is_power_of_2(queue_size)) 591 queue_size = rte_align32pow2(queue_size >> 1); 592 593 if (queue_size == 0) { 594 PMD_INIT_LOG(ERR, "Invalid queue size\n"); 595 return -EFAULT; 596 } 597 598 return queue_size; 599 } 600 601 static void ena_stats_restart(struct rte_eth_dev *dev) 602 { 603 struct ena_adapter *adapter = 604 (struct ena_adapter *)(dev->data->dev_private); 605 606 rte_atomic64_init(&adapter->drv_stats->ierrors); 607 rte_atomic64_init(&adapter->drv_stats->oerrors); 608 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 609 } 610 611 static void ena_stats_get(struct rte_eth_dev *dev, 612 struct rte_eth_stats *stats) 613 { 614 struct ena_admin_basic_stats ena_stats; 615 struct ena_adapter *adapter = 616 (struct ena_adapter *)(dev->data->dev_private); 617 struct ena_com_dev *ena_dev = &adapter->ena_dev; 618 int rc; 619 620 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 621 return; 622 623 memset(&ena_stats, 0, sizeof(ena_stats)); 624 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 625 if (unlikely(rc)) { 626 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 627 return; 628 } 629 630 /* Set of basic statistics from ENA */ 631 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 632 ena_stats.rx_pkts_low); 633 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 634 ena_stats.tx_pkts_low); 635 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 636 ena_stats.rx_bytes_low); 637 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 638 ena_stats.tx_bytes_low); 639 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 640 ena_stats.rx_drops_low); 641 642 /* Driver related stats */ 643 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 644 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 645 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 646 } 647 648 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 649 { 650 struct ena_adapter *adapter; 651 struct ena_com_dev *ena_dev; 652 int rc = 0; 653 654 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 655 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 656 adapter = (struct ena_adapter *)(dev->data->dev_private); 657 658 ena_dev = &adapter->ena_dev; 659 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 660 661 if (mtu > ena_get_mtu_conf(adapter)) { 662 RTE_LOG(ERR, PMD, 663 "Given MTU (%d) exceeds maximum MTU supported (%d)\n", 664 mtu, ena_get_mtu_conf(adapter)); 665 rc = -EINVAL; 666 goto err; 667 } 668 669 rc = ena_com_set_dev_mtu(ena_dev, mtu); 670 if (rc) 671 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 672 else 673 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 674 675 err: 676 return rc; 677 } 678 679 static int ena_start(struct rte_eth_dev *dev) 680 { 681 struct ena_adapter *adapter = 682 (struct ena_adapter *)(dev->data->dev_private); 683 int rc = 0; 684 685 if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || 686 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 687 PMD_INIT_LOG(ERR, "API violation"); 688 return -1; 689 } 690 691 rc = ena_check_valid_conf(adapter); 692 if (rc) 693 return rc; 694 695 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 696 if (rc) 697 return rc; 698 699 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 700 if (rc) 701 return rc; 702 703 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 704 ETH_MQ_RX_RSS_FLAG) { 705 rc = ena_rss_init_default(adapter); 706 if (rc) 707 return rc; 708 } 709 710 ena_stats_restart(dev); 711 712 adapter->state = ENA_ADAPTER_STATE_RUNNING; 713 714 return 0; 715 } 716 717 static int ena_queue_restart(struct ena_ring *ring) 718 { 719 int rc; 720 721 ena_assert_msg(ring->configured == 1, 722 "Trying to restart unconfigured queue\n"); 723 724 ring->next_to_clean = 0; 725 ring->next_to_use = 0; 726 727 if (ring->type == ENA_RING_TYPE_TX) 728 return 0; 729 730 rc = ena_populate_rx_queue(ring, ring->ring_size - 1); 731 if ((unsigned int)rc != ring->ring_size - 1) { 732 PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n"); 733 return (-1); 734 } 735 736 return 0; 737 } 738 739 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 740 uint16_t queue_idx, 741 uint16_t nb_desc, 742 __rte_unused unsigned int socket_id, 743 __rte_unused const struct rte_eth_txconf *tx_conf) 744 { 745 struct ena_com_create_io_ctx ctx = 746 /* policy set to _HOST just to satisfy icc compiler */ 747 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 748 ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 }; 749 struct ena_ring *txq = NULL; 750 struct ena_adapter *adapter = 751 (struct ena_adapter *)(dev->data->dev_private); 752 unsigned int i; 753 int ena_qid; 754 int rc; 755 struct ena_com_dev *ena_dev = &adapter->ena_dev; 756 757 txq = &adapter->tx_ring[queue_idx]; 758 759 if (txq->configured) { 760 RTE_LOG(CRIT, PMD, 761 "API violation. Queue %d is already configured\n", 762 queue_idx); 763 return -1; 764 } 765 766 if (nb_desc > adapter->tx_ring_size) { 767 RTE_LOG(ERR, PMD, 768 "Unsupported size of TX queue (max size: %d)\n", 769 adapter->tx_ring_size); 770 return -EINVAL; 771 } 772 773 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 774 775 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 776 ctx.qid = ena_qid; 777 ctx.msix_vector = -1; /* admin interrupts not used */ 778 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 779 ctx.queue_size = adapter->tx_ring_size; 780 781 rc = ena_com_create_io_queue(ena_dev, &ctx); 782 if (rc) { 783 RTE_LOG(ERR, PMD, 784 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 785 queue_idx, ena_qid, rc); 786 } 787 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 788 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 789 790 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 791 &txq->ena_com_io_sq, 792 &txq->ena_com_io_cq); 793 if (rc) { 794 RTE_LOG(ERR, PMD, 795 "Failed to get TX queue handlers. TX queue num %d rc: %d\n", 796 queue_idx, rc); 797 ena_com_destroy_io_queue(ena_dev, ena_qid); 798 goto err; 799 } 800 801 txq->port_id = dev->data->port_id; 802 txq->next_to_clean = 0; 803 txq->next_to_use = 0; 804 txq->ring_size = nb_desc; 805 806 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 807 sizeof(struct ena_tx_buffer) * 808 txq->ring_size, 809 RTE_CACHE_LINE_SIZE); 810 if (!txq->tx_buffer_info) { 811 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 812 return -ENOMEM; 813 } 814 815 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 816 sizeof(u16) * txq->ring_size, 817 RTE_CACHE_LINE_SIZE); 818 if (!txq->empty_tx_reqs) { 819 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 820 rte_free(txq->tx_buffer_info); 821 return -ENOMEM; 822 } 823 for (i = 0; i < txq->ring_size; i++) 824 txq->empty_tx_reqs[i] = i; 825 826 /* Store pointer to this queue in upper layer */ 827 txq->configured = 1; 828 dev->data->tx_queues[queue_idx] = txq; 829 err: 830 return rc; 831 } 832 833 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 834 uint16_t queue_idx, 835 uint16_t nb_desc, 836 __rte_unused unsigned int socket_id, 837 __rte_unused const struct rte_eth_rxconf *rx_conf, 838 struct rte_mempool *mp) 839 { 840 struct ena_com_create_io_ctx ctx = 841 /* policy set to _HOST just to satisfy icc compiler */ 842 { ENA_ADMIN_PLACEMENT_POLICY_HOST, 843 ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 }; 844 struct ena_adapter *adapter = 845 (struct ena_adapter *)(dev->data->dev_private); 846 struct ena_ring *rxq = NULL; 847 uint16_t ena_qid = 0; 848 int rc = 0; 849 struct ena_com_dev *ena_dev = &adapter->ena_dev; 850 851 rxq = &adapter->rx_ring[queue_idx]; 852 if (rxq->configured) { 853 RTE_LOG(CRIT, PMD, 854 "API violation. Queue %d is already configured\n", 855 queue_idx); 856 return -1; 857 } 858 859 if (nb_desc > adapter->rx_ring_size) { 860 RTE_LOG(ERR, PMD, 861 "Unsupported size of RX queue (max size: %d)\n", 862 adapter->rx_ring_size); 863 return -EINVAL; 864 } 865 866 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 867 868 ctx.qid = ena_qid; 869 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 870 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 871 ctx.msix_vector = -1; /* admin interrupts not used */ 872 ctx.queue_size = adapter->rx_ring_size; 873 874 rc = ena_com_create_io_queue(ena_dev, &ctx); 875 if (rc) 876 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 877 queue_idx, rc); 878 879 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 880 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 881 882 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 883 &rxq->ena_com_io_sq, 884 &rxq->ena_com_io_cq); 885 if (rc) { 886 RTE_LOG(ERR, PMD, 887 "Failed to get RX queue handlers. RX queue num %d rc: %d\n", 888 queue_idx, rc); 889 ena_com_destroy_io_queue(ena_dev, ena_qid); 890 } 891 892 rxq->port_id = dev->data->port_id; 893 rxq->next_to_clean = 0; 894 rxq->next_to_use = 0; 895 rxq->ring_size = nb_desc; 896 rxq->mb_pool = mp; 897 898 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 899 sizeof(struct rte_mbuf *) * nb_desc, 900 RTE_CACHE_LINE_SIZE); 901 if (!rxq->rx_buffer_info) { 902 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 903 return -ENOMEM; 904 } 905 906 /* Store pointer to this queue in upper layer */ 907 rxq->configured = 1; 908 dev->data->rx_queues[queue_idx] = rxq; 909 910 return rc; 911 } 912 913 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 914 { 915 unsigned int i; 916 int rc; 917 unsigned int ring_size = rxq->ring_size; 918 unsigned int ring_mask = ring_size - 1; 919 int next_to_use = rxq->next_to_use & ring_mask; 920 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 921 922 if (unlikely(!count)) 923 return 0; 924 925 ena_assert_msg((((ENA_CIRC_COUNT(rxq->next_to_use, rxq->next_to_clean, 926 rxq->ring_size)) + 927 count) < rxq->ring_size), "bad ring state"); 928 929 count = RTE_MIN(count, ring_size - next_to_use); 930 931 /* get resources for incoming packets */ 932 rc = rte_mempool_get_bulk(rxq->mb_pool, 933 (void **)(&mbufs[next_to_use]), count); 934 if (unlikely(rc < 0)) { 935 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 936 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 937 return 0; 938 } 939 940 for (i = 0; i < count; i++) { 941 struct rte_mbuf *mbuf = mbufs[next_to_use]; 942 struct ena_com_buf ebuf; 943 944 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 945 /* prepare physical address for DMA transaction */ 946 ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; 947 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 948 /* pass resource to device */ 949 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 950 &ebuf, next_to_use); 951 if (unlikely(rc)) { 952 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 953 break; 954 } 955 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, ring_size); 956 } 957 958 rte_wmb(); 959 rxq->next_to_use = next_to_use; 960 /* let HW know that it can fill buffers with data */ 961 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 962 963 return i; 964 } 965 966 static int ena_device_init(struct ena_com_dev *ena_dev, 967 struct ena_com_dev_get_features_ctx *get_feat_ctx) 968 { 969 int rc; 970 971 /* Initialize mmio registers */ 972 rc = ena_com_mmio_reg_read_request_init(ena_dev); 973 if (rc) { 974 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 975 return rc; 976 } 977 978 /* reset device */ 979 rc = ena_com_dev_reset(ena_dev); 980 if (rc) { 981 RTE_LOG(ERR, PMD, "cannot reset device\n"); 982 goto err_mmio_read_less; 983 } 984 985 /* check FW version */ 986 rc = ena_com_validate_version(ena_dev); 987 if (rc) { 988 RTE_LOG(ERR, PMD, "device version is too low\n"); 989 goto err_mmio_read_less; 990 } 991 992 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 993 994 /* ENA device administration layer init */ 995 rc = ena_com_admin_init(ena_dev, NULL, true); 996 if (rc) { 997 RTE_LOG(ERR, PMD, 998 "cannot initialize ena admin queue with device\n"); 999 goto err_mmio_read_less; 1000 } 1001 1002 /* To enable the msix interrupts the driver needs to know the number 1003 * of queues. So the driver uses polling mode to retrieve this 1004 * information. 1005 */ 1006 ena_com_set_admin_polling_mode(ena_dev, true); 1007 1008 /* Get Device Attributes and features */ 1009 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 1010 if (rc) { 1011 RTE_LOG(ERR, PMD, 1012 "cannot get attribute for ena device rc= %d\n", rc); 1013 goto err_admin_init; 1014 } 1015 1016 return 0; 1017 1018 err_admin_init: 1019 ena_com_admin_destroy(ena_dev); 1020 1021 err_mmio_read_less: 1022 ena_com_mmio_reg_read_request_destroy(ena_dev); 1023 1024 return rc; 1025 } 1026 1027 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 1028 { 1029 struct rte_pci_device *pci_dev; 1030 struct ena_adapter *adapter = 1031 (struct ena_adapter *)(eth_dev->data->dev_private); 1032 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1033 struct ena_com_dev_get_features_ctx get_feat_ctx; 1034 int queue_size, rc; 1035 1036 static int adapters_found; 1037 1038 memset(adapter, 0, sizeof(struct ena_adapter)); 1039 ena_dev = &adapter->ena_dev; 1040 1041 eth_dev->dev_ops = &ena_dev_ops; 1042 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1043 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1044 adapter->rte_eth_dev_data = eth_dev->data; 1045 adapter->rte_dev = eth_dev; 1046 1047 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1048 return 0; 1049 1050 pci_dev = eth_dev->pci_dev; 1051 adapter->pdev = pci_dev; 1052 1053 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", 1054 pci_dev->addr.domain, 1055 pci_dev->addr.bus, 1056 pci_dev->addr.devid, 1057 pci_dev->addr.function); 1058 1059 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1060 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1061 1062 /* Present ENA_MEM_BAR indicates available LLQ mode. 1063 * Use corresponding policy 1064 */ 1065 if (adapter->dev_mem_base) 1066 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 1067 else if (adapter->regs) 1068 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1069 else 1070 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", 1071 ENA_REGS_BAR); 1072 1073 ena_dev->reg_bar = adapter->regs; 1074 ena_dev->dmadev = adapter->pdev; 1075 1076 adapter->id_number = adapters_found; 1077 1078 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1079 adapter->id_number); 1080 1081 /* device specific initialization routine */ 1082 rc = ena_device_init(ena_dev, &get_feat_ctx); 1083 if (rc) { 1084 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); 1085 return -1; 1086 } 1087 1088 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1089 if (get_feat_ctx.max_queues.max_llq_num == 0) { 1090 PMD_INIT_LOG(ERR, 1091 "Trying to use LLQ but llq_num is 0.\n" 1092 "Fall back into regular queues.\n"); 1093 ena_dev->tx_mem_queue_type = 1094 ENA_ADMIN_PLACEMENT_POLICY_HOST; 1095 adapter->num_queues = 1096 get_feat_ctx.max_queues.max_sq_num; 1097 } else { 1098 adapter->num_queues = 1099 get_feat_ctx.max_queues.max_llq_num; 1100 } 1101 } else { 1102 adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; 1103 } 1104 1105 queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); 1106 if ((queue_size <= 0) || (adapter->num_queues <= 0)) 1107 return -EFAULT; 1108 1109 adapter->tx_ring_size = queue_size; 1110 adapter->rx_ring_size = queue_size; 1111 1112 /* prepare ring structures */ 1113 ena_init_rings(adapter); 1114 1115 /* Set max MTU for this device */ 1116 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1117 1118 /* Copy MAC address and point DPDK to it */ 1119 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1120 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1121 (struct ether_addr *)adapter->mac_addr); 1122 1123 adapter->drv_stats = rte_zmalloc("adapter stats", 1124 sizeof(*adapter->drv_stats), 1125 RTE_CACHE_LINE_SIZE); 1126 if (!adapter->drv_stats) { 1127 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1128 return -ENOMEM; 1129 } 1130 1131 adapters_found++; 1132 adapter->state = ENA_ADAPTER_STATE_INIT; 1133 1134 return 0; 1135 } 1136 1137 static int ena_dev_configure(struct rte_eth_dev *dev) 1138 { 1139 struct ena_adapter *adapter = 1140 (struct ena_adapter *)(dev->data->dev_private); 1141 1142 if (!(adapter->state == ENA_ADAPTER_STATE_INIT || 1143 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 1144 PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n", 1145 adapter->state); 1146 return -1; 1147 } 1148 1149 switch (adapter->state) { 1150 case ENA_ADAPTER_STATE_INIT: 1151 case ENA_ADAPTER_STATE_STOPPED: 1152 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1153 break; 1154 case ENA_ADAPTER_STATE_CONFIG: 1155 RTE_LOG(WARNING, PMD, 1156 "Ivalid driver state while trying to configure device\n"); 1157 break; 1158 default: 1159 break; 1160 } 1161 1162 return 0; 1163 } 1164 1165 static void ena_init_rings(struct ena_adapter *adapter) 1166 { 1167 int i; 1168 1169 for (i = 0; i < adapter->num_queues; i++) { 1170 struct ena_ring *ring = &adapter->tx_ring[i]; 1171 1172 ring->configured = 0; 1173 ring->type = ENA_RING_TYPE_TX; 1174 ring->adapter = adapter; 1175 ring->id = i; 1176 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1177 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1178 } 1179 1180 for (i = 0; i < adapter->num_queues; i++) { 1181 struct ena_ring *ring = &adapter->rx_ring[i]; 1182 1183 ring->configured = 0; 1184 ring->type = ENA_RING_TYPE_RX; 1185 ring->adapter = adapter; 1186 ring->id = i; 1187 } 1188 } 1189 1190 static void ena_infos_get(struct rte_eth_dev *dev, 1191 struct rte_eth_dev_info *dev_info) 1192 { 1193 struct ena_adapter *adapter; 1194 struct ena_com_dev *ena_dev; 1195 struct ena_com_dev_get_features_ctx feat; 1196 uint32_t rx_feat = 0, tx_feat = 0; 1197 int rc = 0; 1198 1199 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1200 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1201 adapter = (struct ena_adapter *)(dev->data->dev_private); 1202 1203 ena_dev = &adapter->ena_dev; 1204 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1205 1206 dev_info->speed_capa = 1207 ETH_LINK_SPEED_1G | 1208 ETH_LINK_SPEED_2_5G | 1209 ETH_LINK_SPEED_5G | 1210 ETH_LINK_SPEED_10G | 1211 ETH_LINK_SPEED_25G | 1212 ETH_LINK_SPEED_40G | 1213 ETH_LINK_SPEED_50G | 1214 ETH_LINK_SPEED_100G; 1215 1216 /* Get supported features from HW */ 1217 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1218 if (unlikely(rc)) { 1219 RTE_LOG(ERR, PMD, 1220 "Cannot get attribute for ena device rc= %d\n", rc); 1221 return; 1222 } 1223 1224 /* Set Tx & Rx features available for device */ 1225 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1226 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1227 1228 if (feat.offload.tx & 1229 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1230 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1231 DEV_TX_OFFLOAD_UDP_CKSUM | 1232 DEV_TX_OFFLOAD_TCP_CKSUM; 1233 1234 if (feat.offload.tx & 1235 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1236 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1237 DEV_RX_OFFLOAD_UDP_CKSUM | 1238 DEV_RX_OFFLOAD_TCP_CKSUM; 1239 1240 /* Inform framework about available features */ 1241 dev_info->rx_offload_capa = rx_feat; 1242 dev_info->tx_offload_capa = tx_feat; 1243 1244 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1245 dev_info->max_rx_pktlen = adapter->max_mtu; 1246 dev_info->max_mac_addrs = 1; 1247 1248 dev_info->max_rx_queues = adapter->num_queues; 1249 dev_info->max_tx_queues = adapter->num_queues; 1250 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1251 } 1252 1253 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1254 uint16_t nb_pkts) 1255 { 1256 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1257 unsigned int ring_size = rx_ring->ring_size; 1258 unsigned int ring_mask = ring_size - 1; 1259 uint16_t next_to_clean = rx_ring->next_to_clean; 1260 int desc_in_use = 0; 1261 unsigned int recv_idx = 0; 1262 struct rte_mbuf *mbuf = NULL; 1263 struct rte_mbuf *mbuf_head = NULL; 1264 struct rte_mbuf *mbuf_prev = NULL; 1265 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1266 unsigned int completed; 1267 1268 struct ena_com_rx_ctx ena_rx_ctx; 1269 int rc = 0; 1270 1271 /* Check adapter state */ 1272 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1273 RTE_LOG(ALERT, PMD, 1274 "Trying to receive pkts while device is NOT running\n"); 1275 return 0; 1276 } 1277 1278 desc_in_use = ENA_CIRC_COUNT(rx_ring->next_to_use, 1279 next_to_clean, ring_size); 1280 if (unlikely(nb_pkts > desc_in_use)) 1281 nb_pkts = desc_in_use; 1282 1283 for (completed = 0; completed < nb_pkts; completed++) { 1284 int segments = 0; 1285 1286 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1287 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1288 ena_rx_ctx.descs = 0; 1289 /* receive packet context */ 1290 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1291 rx_ring->ena_com_io_sq, 1292 &ena_rx_ctx); 1293 if (unlikely(rc)) { 1294 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1295 return 0; 1296 } 1297 1298 if (unlikely(ena_rx_ctx.descs == 0)) 1299 break; 1300 1301 while (segments < ena_rx_ctx.descs) { 1302 mbuf = rx_buff_info[next_to_clean & ring_mask]; 1303 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1304 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1305 mbuf->refcnt = 1; 1306 mbuf->next = NULL; 1307 if (segments == 0) { 1308 mbuf->nb_segs = ena_rx_ctx.descs; 1309 mbuf->port = rx_ring->port_id; 1310 mbuf->pkt_len = 0; 1311 mbuf_head = mbuf; 1312 } else { 1313 /* for multi-segment pkts create mbuf chain */ 1314 mbuf_prev->next = mbuf; 1315 } 1316 mbuf_head->pkt_len += mbuf->data_len; 1317 1318 mbuf_prev = mbuf; 1319 segments++; 1320 next_to_clean = 1321 ENA_RX_RING_IDX_NEXT(next_to_clean, ring_size); 1322 } 1323 1324 /* fill mbuf attributes if any */ 1325 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1326 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1327 1328 /* pass to DPDK application head mbuf */ 1329 rx_pkts[recv_idx] = mbuf_head; 1330 recv_idx++; 1331 } 1332 1333 /* Burst refill to save doorbells, memory barriers, const interval */ 1334 if (ring_size - desc_in_use - 1 > ENA_RING_DESCS_RATIO(ring_size)) 1335 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use - 1); 1336 1337 rx_ring->next_to_clean = next_to_clean & ring_mask; 1338 1339 return recv_idx; 1340 } 1341 1342 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1343 uint16_t nb_pkts) 1344 { 1345 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1346 unsigned int next_to_use = tx_ring->next_to_use; 1347 struct rte_mbuf *mbuf; 1348 unsigned int ring_size = tx_ring->ring_size; 1349 unsigned int ring_mask = ring_size - 1; 1350 struct ena_com_tx_ctx ena_tx_ctx; 1351 struct ena_tx_buffer *tx_info; 1352 struct ena_com_buf *ebuf; 1353 uint16_t rc, req_id, total_tx_descs = 0; 1354 int sent_idx = 0; 1355 int nb_hw_desc; 1356 1357 /* Check adapter state */ 1358 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1359 RTE_LOG(ALERT, PMD, 1360 "Trying to xmit pkts while device is NOT running\n"); 1361 return 0; 1362 } 1363 1364 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 1365 mbuf = tx_pkts[sent_idx]; 1366 1367 req_id = tx_ring->empty_tx_reqs[next_to_use]; 1368 tx_info = &tx_ring->tx_buffer_info[req_id]; 1369 tx_info->mbuf = mbuf; 1370 tx_info->num_of_bufs = 0; 1371 ebuf = tx_info->bufs; 1372 1373 /* Prepare TX context */ 1374 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1375 memset(&ena_tx_ctx.ena_meta, 0x0, 1376 sizeof(struct ena_com_tx_meta)); 1377 ena_tx_ctx.ena_bufs = ebuf; 1378 ena_tx_ctx.req_id = req_id; 1379 if (tx_ring->tx_mem_queue_type == 1380 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1381 /* prepare the push buffer with 1382 * virtual address of the data 1383 */ 1384 ena_tx_ctx.header_len = 1385 RTE_MIN(mbuf->data_len, 1386 tx_ring->tx_max_header_size); 1387 ena_tx_ctx.push_header = 1388 (void *)((char *)mbuf->buf_addr + 1389 mbuf->data_off); 1390 } /* there's no else as we take advantage of memset zeroing */ 1391 1392 /* Set TX offloads flags, if applicable */ 1393 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); 1394 1395 if (unlikely(mbuf->ol_flags & 1396 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 1397 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 1398 1399 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 1400 1401 /* Process first segment taking into 1402 * consideration pushed header 1403 */ 1404 if (mbuf->data_len > ena_tx_ctx.header_len) { 1405 ebuf->paddr = mbuf->buf_physaddr + 1406 mbuf->data_off + 1407 ena_tx_ctx.header_len; 1408 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 1409 ebuf++; 1410 tx_info->num_of_bufs++; 1411 } 1412 1413 while ((mbuf = mbuf->next) != NULL) { 1414 ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; 1415 ebuf->len = mbuf->data_len; 1416 ebuf++; 1417 tx_info->num_of_bufs++; 1418 } 1419 1420 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1421 1422 /* Write data to device */ 1423 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 1424 &ena_tx_ctx, &nb_hw_desc); 1425 if (unlikely(rc)) 1426 break; 1427 1428 tx_info->tx_descs = nb_hw_desc; 1429 1430 next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, ring_size); 1431 } 1432 1433 /* Let HW do it's best :-) */ 1434 rte_wmb(); 1435 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 1436 1437 /* Clear complete packets */ 1438 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 1439 /* Get Tx info & store how many descs were processed */ 1440 tx_info = &tx_ring->tx_buffer_info[req_id]; 1441 total_tx_descs += tx_info->tx_descs; 1442 1443 /* Free whole mbuf chain */ 1444 mbuf = tx_info->mbuf; 1445 rte_pktmbuf_free(mbuf); 1446 1447 /* Put back descriptor to the ring for reuse */ 1448 tx_ring->empty_tx_reqs[tx_ring->next_to_clean] = req_id; 1449 tx_ring->next_to_clean = 1450 ENA_TX_RING_IDX_NEXT(tx_ring->next_to_clean, 1451 tx_ring->ring_size); 1452 1453 /* If too many descs to clean, leave it for another run */ 1454 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 1455 break; 1456 } 1457 1458 /* acknowledge completion of sent packets */ 1459 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 1460 tx_ring->next_to_use = next_to_use; 1461 return sent_idx; 1462 } 1463 1464 static struct eth_driver rte_ena_pmd = { 1465 { 1466 .name = "rte_ena_pmd", 1467 .id_table = pci_id_ena_map, 1468 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1469 }, 1470 .eth_dev_init = eth_ena_dev_init, 1471 .dev_private_size = sizeof(struct ena_adapter), 1472 }; 1473 1474 static int 1475 rte_ena_pmd_init(const char *name __rte_unused, 1476 const char *params __rte_unused) 1477 { 1478 rte_eth_driver_register(&rte_ena_pmd); 1479 return 0; 1480 }; 1481 1482 struct rte_driver ena_pmd_drv = { 1483 .type = PMD_PDEV, 1484 .init = rte_ena_pmd_init, 1485 }; 1486 1487 PMD_REGISTER_DRIVER(ena_pmd_drv, ena); 1488 DRIVER_REGISTER_PCI_TABLE(ena, pci_id_ena_map); 1489