1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev.h> 36 #include <rte_tcp.h> 37 #include <rte_atomic.h> 38 #include <rte_dev.h> 39 #include <rte_errno.h> 40 41 #include "ena_ethdev.h" 42 #include "ena_logs.h" 43 #include "ena_platform.h" 44 #include "ena_com.h" 45 #include "ena_eth_com.h" 46 47 #include <ena_common_defs.h> 48 #include <ena_regs_defs.h> 49 #include <ena_admin_defs.h> 50 #include <ena_eth_io_defs.h> 51 52 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 53 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 54 /*reverse version of ENA_IO_RXQ_IDX*/ 55 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 56 57 /* While processing submitted and completed descriptors (rx and tx path 58 * respectively) in a loop it is desired to: 59 * - perform batch submissions while populating sumbissmion queue 60 * - avoid blocking transmission of other packets during cleanup phase 61 * Hence the utilization ratio of 1/8 of a queue size. 62 */ 63 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 64 65 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 66 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 67 68 #define GET_L4_HDR_LEN(mbuf) \ 69 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 70 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 71 72 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 73 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 74 #define ENA_HASH_KEY_SIZE 40 75 76 /** Vendor ID used by Amazon devices */ 77 #define PCI_VENDOR_ID_AMAZON 0x1D0F 78 /** Amazon devices */ 79 #define PCI_DEVICE_ID_ENA_VF 0xEC20 80 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 81 82 static struct rte_pci_id pci_id_ena_map[] = { 83 #define RTE_PCI_DEV_ID_DECL_ENA(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, 84 85 RTE_PCI_DEV_ID_DECL_ENA(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) 86 RTE_PCI_DEV_ID_DECL_ENA(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) 87 {.device_id = 0}, 88 }; 89 90 static int ena_device_init(struct ena_com_dev *ena_dev, 91 struct ena_com_dev_get_features_ctx *get_feat_ctx); 92 static int ena_dev_configure(struct rte_eth_dev *dev); 93 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 94 uint16_t nb_pkts); 95 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 96 uint16_t nb_desc, unsigned int socket_id, 97 const struct rte_eth_txconf *tx_conf); 98 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 99 uint16_t nb_desc, unsigned int socket_id, 100 const struct rte_eth_rxconf *rx_conf, 101 struct rte_mempool *mp); 102 static uint16_t eth_ena_recv_pkts(void *rx_queue, 103 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 104 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 105 static void ena_init_rings(struct ena_adapter *adapter); 106 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 107 static int ena_start(struct rte_eth_dev *dev); 108 static void ena_close(struct rte_eth_dev *dev); 109 static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 110 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 111 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 112 static void ena_rx_queue_release(void *queue); 113 static void ena_tx_queue_release(void *queue); 114 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 115 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 116 static int ena_link_update(struct rte_eth_dev *dev, 117 __rte_unused int wait_to_complete); 118 static int ena_queue_restart(struct ena_ring *ring); 119 static int ena_queue_restart_all(struct rte_eth_dev *dev, 120 enum ena_ring_type ring_type); 121 static void ena_stats_restart(struct rte_eth_dev *dev); 122 static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, 123 struct rte_eth_dev_info *dev_info); 124 static int ena_rss_reta_update(struct rte_eth_dev *dev, 125 struct rte_eth_rss_reta_entry64 *reta_conf, 126 uint16_t reta_size); 127 static int ena_rss_reta_query(struct rte_eth_dev *dev, 128 struct rte_eth_rss_reta_entry64 *reta_conf, 129 uint16_t reta_size); 130 131 static struct eth_dev_ops ena_dev_ops = { 132 .dev_configure = ena_dev_configure, 133 .dev_infos_get = ena_infos_get, 134 .rx_queue_setup = ena_rx_queue_setup, 135 .tx_queue_setup = ena_tx_queue_setup, 136 .dev_start = ena_start, 137 .link_update = ena_link_update, 138 .stats_get = ena_stats_get, 139 .mtu_set = ena_mtu_set, 140 .rx_queue_release = ena_rx_queue_release, 141 .tx_queue_release = ena_tx_queue_release, 142 .dev_close = ena_close, 143 .reta_update = ena_rss_reta_update, 144 .reta_query = ena_rss_reta_query, 145 }; 146 147 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 148 struct ena_com_rx_ctx *ena_rx_ctx) 149 { 150 uint64_t ol_flags = 0; 151 152 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 153 ol_flags |= PKT_TX_TCP_CKSUM; 154 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 155 ol_flags |= PKT_TX_UDP_CKSUM; 156 157 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 158 ol_flags |= PKT_TX_IPV4; 159 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 160 ol_flags |= PKT_TX_IPV6; 161 162 if (unlikely(ena_rx_ctx->l4_csum_err)) 163 ol_flags |= PKT_RX_L4_CKSUM_BAD; 164 if (unlikely(ena_rx_ctx->l3_csum_err)) 165 ol_flags |= PKT_RX_IP_CKSUM_BAD; 166 167 mbuf->ol_flags = ol_flags; 168 } 169 170 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 171 struct ena_com_tx_ctx *ena_tx_ctx) 172 { 173 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 174 175 if (mbuf->ol_flags & 176 (PKT_TX_L4_MASK | PKT_TX_IP_CKSUM | PKT_TX_TCP_SEG)) { 177 /* check if TSO is required */ 178 if (mbuf->ol_flags & PKT_TX_TCP_SEG) { 179 ena_tx_ctx->tso_enable = true; 180 181 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 182 } 183 184 /* check if L3 checksum is needed */ 185 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 186 ena_tx_ctx->l3_csum_enable = true; 187 188 if (mbuf->ol_flags & PKT_TX_IPV6) { 189 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 190 } else { 191 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 192 193 /* set don't fragment (DF) flag */ 194 if (mbuf->packet_type & 195 (RTE_PTYPE_L4_NONFRAG 196 | RTE_PTYPE_INNER_L4_NONFRAG)) 197 ena_tx_ctx->df = true; 198 } 199 200 /* check if L4 checksum is needed */ 201 switch (mbuf->ol_flags & PKT_TX_L4_MASK) { 202 case PKT_TX_TCP_CKSUM: 203 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 204 ena_tx_ctx->l4_csum_enable = true; 205 break; 206 case PKT_TX_UDP_CKSUM: 207 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 208 ena_tx_ctx->l4_csum_enable = true; 209 break; 210 default: 211 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 212 ena_tx_ctx->l4_csum_enable = false; 213 break; 214 } 215 216 ena_meta->mss = mbuf->tso_segsz; 217 ena_meta->l3_hdr_len = mbuf->l3_len; 218 ena_meta->l3_hdr_offset = mbuf->l2_len; 219 /* this param needed only for TSO */ 220 ena_meta->l3_outer_hdr_len = 0; 221 ena_meta->l3_outer_hdr_offset = 0; 222 223 ena_tx_ctx->meta_valid = true; 224 } else { 225 ena_tx_ctx->meta_valid = false; 226 } 227 } 228 229 static void ena_close(struct rte_eth_dev *dev) 230 { 231 struct ena_adapter *adapter = 232 (struct ena_adapter *)(dev->data->dev_private); 233 234 adapter->state = ENA_ADAPTER_STATE_STOPPED; 235 236 ena_rx_queue_release_all(dev); 237 ena_tx_queue_release_all(dev); 238 } 239 240 static int ena_rss_reta_update(struct rte_eth_dev *dev, 241 struct rte_eth_rss_reta_entry64 *reta_conf, 242 uint16_t reta_size) 243 { 244 struct ena_adapter *adapter = 245 (struct ena_adapter *)(dev->data->dev_private); 246 struct ena_com_dev *ena_dev = &adapter->ena_dev; 247 int ret, i; 248 u16 entry_value; 249 int conf_idx; 250 int idx; 251 252 if ((reta_size == 0) || (reta_conf == NULL)) 253 return -EINVAL; 254 255 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 256 RTE_LOG(WARNING, PMD, 257 "indirection table %d is bigger than supported (%d)\n", 258 reta_size, ENA_RX_RSS_TABLE_SIZE); 259 ret = -EINVAL; 260 goto err; 261 } 262 263 for (i = 0 ; i < reta_size ; i++) { 264 /* each reta_conf is for 64 entries. 265 * to support 128 we use 2 conf of 64 266 */ 267 conf_idx = i / RTE_RETA_GROUP_SIZE; 268 idx = i % RTE_RETA_GROUP_SIZE; 269 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 270 entry_value = 271 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 272 ret = ena_com_indirect_table_fill_entry(ena_dev, 273 i, 274 entry_value); 275 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 276 RTE_LOG(ERR, PMD, 277 "Cannot fill indirect table\n"); 278 ret = -ENOTSUP; 279 goto err; 280 } 281 } 282 } 283 284 ret = ena_com_indirect_table_set(ena_dev); 285 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 286 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 287 ret = -ENOTSUP; 288 goto err; 289 } 290 291 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 292 __func__, reta_size, adapter->rte_dev->data->port_id); 293 err: 294 return ret; 295 } 296 297 /* Query redirection table. */ 298 static int ena_rss_reta_query(struct rte_eth_dev *dev, 299 struct rte_eth_rss_reta_entry64 *reta_conf, 300 uint16_t reta_size) 301 { 302 struct ena_adapter *adapter = 303 (struct ena_adapter *)(dev->data->dev_private); 304 struct ena_com_dev *ena_dev = &adapter->ena_dev; 305 int ret; 306 int i; 307 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 308 int reta_conf_idx; 309 int reta_idx; 310 311 if (reta_size == 0 || reta_conf == NULL || 312 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 313 return -EINVAL; 314 315 ret = ena_com_indirect_table_get(ena_dev, indirect_table); 316 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 317 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 318 ret = -ENOTSUP; 319 goto err; 320 } 321 322 for (i = 0 ; i < reta_size ; i++) { 323 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 324 reta_idx = i % RTE_RETA_GROUP_SIZE; 325 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 326 reta_conf[reta_conf_idx].reta[reta_idx] = 327 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 328 } 329 err: 330 return ret; 331 } 332 333 static int ena_rss_init_default(struct ena_adapter *adapter) 334 { 335 struct ena_com_dev *ena_dev = &adapter->ena_dev; 336 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 337 int rc, i; 338 u32 val; 339 340 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 341 if (unlikely(rc)) { 342 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 343 goto err_rss_init; 344 } 345 346 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 347 val = i % nb_rx_queues; 348 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 349 ENA_IO_RXQ_IDX(val)); 350 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 351 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 352 goto err_fill_indir; 353 } 354 } 355 356 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 357 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 358 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 359 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 360 goto err_fill_indir; 361 } 362 363 rc = ena_com_set_default_hash_ctrl(ena_dev); 364 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 365 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 366 goto err_fill_indir; 367 } 368 369 rc = ena_com_indirect_table_set(ena_dev); 370 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 371 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 372 goto err_fill_indir; 373 } 374 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 375 adapter->rte_dev->data->port_id); 376 377 return 0; 378 379 err_fill_indir: 380 ena_com_rss_destroy(ena_dev); 381 err_rss_init: 382 383 return rc; 384 } 385 386 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 387 { 388 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 389 int nb_queues = dev->data->nb_rx_queues; 390 int i; 391 392 for (i = 0; i < nb_queues; i++) 393 ena_rx_queue_release(queues[i]); 394 } 395 396 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 397 { 398 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 399 int nb_queues = dev->data->nb_tx_queues; 400 int i; 401 402 for (i = 0; i < nb_queues; i++) 403 ena_tx_queue_release(queues[i]); 404 } 405 406 static void ena_rx_queue_release(void *queue) 407 { 408 struct ena_ring *ring = (struct ena_ring *)queue; 409 struct ena_adapter *adapter = ring->adapter; 410 int ena_qid; 411 412 ena_assert_msg(ring->configured, 413 "API violation - releasing not configured queue"); 414 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 415 "API violation"); 416 417 /* Destroy HW queue */ 418 ena_qid = ENA_IO_RXQ_IDX(ring->id); 419 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 420 421 /* Free all bufs */ 422 ena_rx_queue_release_bufs(ring); 423 424 /* Free ring resources */ 425 if (ring->rx_buffer_info) 426 rte_free(ring->rx_buffer_info); 427 ring->rx_buffer_info = NULL; 428 429 ring->configured = 0; 430 431 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 432 ring->port_id, ring->id); 433 } 434 435 static void ena_tx_queue_release(void *queue) 436 { 437 struct ena_ring *ring = (struct ena_ring *)queue; 438 struct ena_adapter *adapter = ring->adapter; 439 int ena_qid; 440 441 ena_assert_msg(ring->configured, 442 "API violation. Releasing not configured queue"); 443 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 444 "API violation"); 445 446 /* Destroy HW queue */ 447 ena_qid = ENA_IO_TXQ_IDX(ring->id); 448 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 449 450 /* Free all bufs */ 451 ena_tx_queue_release_bufs(ring); 452 453 /* Free ring resources */ 454 if (ring->tx_buffer_info) 455 rte_free(ring->tx_buffer_info); 456 457 if (ring->empty_tx_reqs) 458 rte_free(ring->empty_tx_reqs); 459 460 ring->empty_tx_reqs = NULL; 461 ring->tx_buffer_info = NULL; 462 463 ring->configured = 0; 464 465 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 466 ring->port_id, ring->id); 467 } 468 469 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 470 { 471 unsigned int ring_mask = ring->ring_size - 1; 472 473 while (ring->next_to_clean != ring->next_to_use) { 474 struct rte_mbuf *m = 475 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 476 477 if (m) 478 __rte_mbuf_raw_free(m); 479 480 ring->next_to_clean = 481 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 482 } 483 } 484 485 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 486 { 487 unsigned int ring_mask = ring->ring_size - 1; 488 489 while (ring->next_to_clean != ring->next_to_use) { 490 struct ena_tx_buffer *tx_buf = 491 &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; 492 493 if (tx_buf->mbuf) 494 rte_pktmbuf_free(tx_buf->mbuf); 495 496 ring->next_to_clean = 497 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 498 } 499 } 500 501 static int ena_link_update(struct rte_eth_dev *dev, 502 __rte_unused int wait_to_complete) 503 { 504 struct rte_eth_link *link = &dev->data->dev_link; 505 506 link->link_status = 1; 507 link->link_speed = ETH_SPEED_NUM_10G; 508 link->link_duplex = ETH_LINK_FULL_DUPLEX; 509 510 return 0; 511 } 512 513 static int ena_queue_restart_all(struct rte_eth_dev *dev, 514 enum ena_ring_type ring_type) 515 { 516 struct ena_adapter *adapter = 517 (struct ena_adapter *)(dev->data->dev_private); 518 struct ena_ring *queues = NULL; 519 int i = 0; 520 int rc = 0; 521 522 queues = (ring_type == ENA_RING_TYPE_RX) ? 523 adapter->rx_ring : adapter->tx_ring; 524 525 for (i = 0; i < adapter->num_queues; i++) { 526 if (queues[i].configured) { 527 if (ring_type == ENA_RING_TYPE_RX) { 528 ena_assert_msg( 529 dev->data->rx_queues[i] == &queues[i], 530 "Inconsistent state of rx queues\n"); 531 } else { 532 ena_assert_msg( 533 dev->data->tx_queues[i] == &queues[i], 534 "Inconsistent state of tx queues\n"); 535 } 536 537 rc = ena_queue_restart(&queues[i]); 538 539 if (rc) { 540 PMD_INIT_LOG(ERR, 541 "failed to restart queue %d type(%d)\n", 542 i, ring_type); 543 return -1; 544 } 545 } 546 } 547 548 return 0; 549 } 550 551 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 552 { 553 uint32_t max_frame_len = adapter->max_mtu; 554 555 if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) 556 max_frame_len = 557 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 558 559 return max_frame_len; 560 } 561 562 static int ena_check_valid_conf(struct ena_adapter *adapter) 563 { 564 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 565 566 if (max_frame_len > adapter->max_mtu) { 567 PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n", max_frame_len); 568 return -1; 569 } 570 571 return 0; 572 } 573 574 static int 575 ena_calc_queue_size(struct ena_com_dev *ena_dev, 576 struct ena_com_dev_get_features_ctx *get_feat_ctx) 577 { 578 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 579 580 queue_size = RTE_MIN(queue_size, 581 get_feat_ctx->max_queues.max_cq_depth); 582 queue_size = RTE_MIN(queue_size, 583 get_feat_ctx->max_queues.max_sq_depth); 584 585 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 586 queue_size = RTE_MIN(queue_size, 587 get_feat_ctx->max_queues.max_llq_depth); 588 589 /* Round down to power of 2 */ 590 if (!rte_is_power_of_2(queue_size)) 591 queue_size = rte_align32pow2(queue_size >> 1); 592 593 if (queue_size == 0) { 594 PMD_INIT_LOG(ERR, "Invalid queue size\n"); 595 return -EFAULT; 596 } 597 598 return queue_size; 599 } 600 601 static void ena_stats_restart(struct rte_eth_dev *dev) 602 { 603 struct ena_adapter *adapter = 604 (struct ena_adapter *)(dev->data->dev_private); 605 606 rte_atomic64_init(&adapter->drv_stats->ierrors); 607 rte_atomic64_init(&adapter->drv_stats->oerrors); 608 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 609 } 610 611 static void ena_stats_get(struct rte_eth_dev *dev, 612 struct rte_eth_stats *stats) 613 { 614 struct ena_admin_basic_stats ena_stats; 615 struct ena_adapter *adapter = 616 (struct ena_adapter *)(dev->data->dev_private); 617 struct ena_com_dev *ena_dev = &adapter->ena_dev; 618 int rc; 619 620 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 621 return; 622 623 memset(&ena_stats, 0, sizeof(ena_stats)); 624 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 625 if (unlikely(rc)) { 626 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 627 return; 628 } 629 630 /* Set of basic statistics from ENA */ 631 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 632 ena_stats.rx_pkts_low); 633 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 634 ena_stats.tx_pkts_low); 635 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 636 ena_stats.rx_bytes_low); 637 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 638 ena_stats.tx_bytes_low); 639 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 640 ena_stats.rx_drops_low); 641 642 /* Driver related stats */ 643 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 644 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 645 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 646 } 647 648 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 649 { 650 struct ena_adapter *adapter; 651 struct ena_com_dev *ena_dev; 652 int rc = 0; 653 654 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 655 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 656 adapter = (struct ena_adapter *)(dev->data->dev_private); 657 658 ena_dev = &adapter->ena_dev; 659 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 660 661 if (mtu > ena_get_mtu_conf(adapter)) { 662 RTE_LOG(ERR, PMD, 663 "Given MTU (%d) exceeds maximum MTU supported (%d)\n", 664 mtu, ena_get_mtu_conf(adapter)); 665 rc = -EINVAL; 666 goto err; 667 } 668 669 rc = ena_com_set_dev_mtu(ena_dev, mtu); 670 if (rc) 671 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 672 else 673 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 674 675 err: 676 return rc; 677 } 678 679 static int ena_start(struct rte_eth_dev *dev) 680 { 681 struct ena_adapter *adapter = 682 (struct ena_adapter *)(dev->data->dev_private); 683 int rc = 0; 684 685 if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || 686 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 687 PMD_INIT_LOG(ERR, "API violation"); 688 return -1; 689 } 690 691 rc = ena_check_valid_conf(adapter); 692 if (rc) 693 return rc; 694 695 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 696 if (rc) 697 return rc; 698 699 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 700 if (rc) 701 return rc; 702 703 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 704 ETH_MQ_RX_RSS_FLAG) { 705 rc = ena_rss_init_default(adapter); 706 if (rc) 707 return rc; 708 } 709 710 ena_stats_restart(dev); 711 712 adapter->state = ENA_ADAPTER_STATE_RUNNING; 713 714 return 0; 715 } 716 717 static int ena_queue_restart(struct ena_ring *ring) 718 { 719 int rc; 720 721 ena_assert_msg(ring->configured == 1, 722 "Trying to restart unconfigured queue\n"); 723 724 ring->next_to_clean = 0; 725 ring->next_to_use = 0; 726 727 if (ring->type == ENA_RING_TYPE_TX) 728 return 0; 729 730 rc = ena_populate_rx_queue(ring, ring->ring_size - 1); 731 if ((unsigned int)rc != ring->ring_size - 1) { 732 PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n"); 733 return (-1); 734 } 735 736 return 0; 737 } 738 739 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 740 uint16_t queue_idx, 741 uint16_t nb_desc, 742 __rte_unused unsigned int socket_id, 743 __rte_unused const struct rte_eth_txconf *tx_conf) 744 { 745 struct ena_ring *txq = NULL; 746 struct ena_adapter *adapter = 747 (struct ena_adapter *)(dev->data->dev_private); 748 unsigned int i; 749 int ena_qid; 750 int rc; 751 struct ena_com_dev *ena_dev = &adapter->ena_dev; 752 753 txq = &adapter->tx_ring[queue_idx]; 754 755 if (txq->configured) { 756 RTE_LOG(CRIT, PMD, 757 "API violation. Queue %d is already configured\n", 758 queue_idx); 759 return -1; 760 } 761 762 if (nb_desc > adapter->tx_ring_size) { 763 RTE_LOG(ERR, PMD, 764 "Unsupported size of TX queue (max size: %d)\n", 765 adapter->tx_ring_size); 766 return -EINVAL; 767 } 768 769 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 770 rc = ena_com_create_io_queue(ena_dev, ena_qid, 771 ENA_COM_IO_QUEUE_DIRECTION_TX, 772 ena_dev->tx_mem_queue_type, 773 -1 /* admin interrupts is not used */, 774 nb_desc); 775 if (rc) { 776 RTE_LOG(ERR, PMD, 777 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 778 queue_idx, ena_qid, rc); 779 } 780 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 781 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 782 783 txq->port_id = dev->data->port_id; 784 txq->next_to_clean = 0; 785 txq->next_to_use = 0; 786 txq->ring_size = nb_desc; 787 788 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 789 sizeof(struct ena_tx_buffer) * 790 txq->ring_size, 791 RTE_CACHE_LINE_SIZE); 792 if (!txq->tx_buffer_info) { 793 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 794 return -ENOMEM; 795 } 796 797 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 798 sizeof(u16) * txq->ring_size, 799 RTE_CACHE_LINE_SIZE); 800 if (!txq->empty_tx_reqs) { 801 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 802 rte_free(txq->tx_buffer_info); 803 return -ENOMEM; 804 } 805 for (i = 0; i < txq->ring_size; i++) 806 txq->empty_tx_reqs[i] = i; 807 808 /* Store pointer to this queue in upper layer */ 809 txq->configured = 1; 810 dev->data->tx_queues[queue_idx] = txq; 811 812 return rc; 813 } 814 815 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 816 uint16_t queue_idx, 817 uint16_t nb_desc, 818 __rte_unused unsigned int socket_id, 819 __rte_unused const struct rte_eth_rxconf *rx_conf, 820 struct rte_mempool *mp) 821 { 822 struct ena_adapter *adapter = 823 (struct ena_adapter *)(dev->data->dev_private); 824 struct ena_ring *rxq = NULL; 825 uint16_t ena_qid = 0; 826 int rc = 0; 827 struct ena_com_dev *ena_dev = &adapter->ena_dev; 828 829 rxq = &adapter->rx_ring[queue_idx]; 830 if (rxq->configured) { 831 RTE_LOG(CRIT, PMD, 832 "API violation. Queue %d is already configured\n", 833 queue_idx); 834 return -1; 835 } 836 837 if (nb_desc > adapter->rx_ring_size) { 838 RTE_LOG(ERR, PMD, 839 "Unsupported size of RX queue (max size: %d)\n", 840 adapter->rx_ring_size); 841 return -EINVAL; 842 } 843 844 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 845 rc = ena_com_create_io_queue(ena_dev, ena_qid, 846 ENA_COM_IO_QUEUE_DIRECTION_RX, 847 ENA_ADMIN_PLACEMENT_POLICY_HOST, 848 -1 /* admin interrupts not used */, 849 nb_desc); 850 if (rc) 851 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 852 queue_idx, rc); 853 854 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 855 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 856 857 rxq->port_id = dev->data->port_id; 858 rxq->next_to_clean = 0; 859 rxq->next_to_use = 0; 860 rxq->ring_size = nb_desc; 861 rxq->mb_pool = mp; 862 863 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 864 sizeof(struct rte_mbuf *) * nb_desc, 865 RTE_CACHE_LINE_SIZE); 866 if (!rxq->rx_buffer_info) { 867 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 868 return -ENOMEM; 869 } 870 871 /* Store pointer to this queue in upper layer */ 872 rxq->configured = 1; 873 dev->data->rx_queues[queue_idx] = rxq; 874 875 return rc; 876 } 877 878 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 879 { 880 unsigned int i; 881 int rc; 882 unsigned int ring_size = rxq->ring_size; 883 unsigned int ring_mask = ring_size - 1; 884 int next_to_use = rxq->next_to_use & ring_mask; 885 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 886 887 if (unlikely(!count)) 888 return 0; 889 890 ena_assert_msg((((ENA_CIRC_COUNT(rxq->next_to_use, rxq->next_to_clean, 891 rxq->ring_size)) + 892 count) < rxq->ring_size), "bad ring state"); 893 894 count = RTE_MIN(count, ring_size - next_to_use); 895 896 /* get resources for incoming packets */ 897 rc = rte_mempool_get_bulk(rxq->mb_pool, 898 (void **)(&mbufs[next_to_use]), count); 899 if (unlikely(rc < 0)) { 900 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 901 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 902 return 0; 903 } 904 905 for (i = 0; i < count; i++) { 906 struct rte_mbuf *mbuf = mbufs[next_to_use]; 907 struct ena_com_buf ebuf; 908 909 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 910 /* prepare physical address for DMA transaction */ 911 ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; 912 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 913 /* pass resource to device */ 914 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 915 &ebuf, next_to_use); 916 if (unlikely(rc)) { 917 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 918 break; 919 } 920 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, ring_size); 921 } 922 923 rte_wmb(); 924 rxq->next_to_use = next_to_use; 925 /* let HW know that it can fill buffers with data */ 926 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 927 928 return i; 929 } 930 931 static int ena_device_init(struct ena_com_dev *ena_dev, 932 struct ena_com_dev_get_features_ctx *get_feat_ctx) 933 { 934 int rc; 935 936 /* Initialize mmio registers */ 937 rc = ena_com_mmio_reg_read_request_init(ena_dev); 938 if (rc) { 939 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 940 return rc; 941 } 942 943 /* reset device */ 944 rc = ena_com_dev_reset(ena_dev); 945 if (rc) { 946 RTE_LOG(ERR, PMD, "cannot reset device\n"); 947 goto err_mmio_read_less; 948 } 949 950 /* check FW version */ 951 rc = ena_com_validate_version(ena_dev); 952 if (rc) { 953 RTE_LOG(ERR, PMD, "device version is too low\n"); 954 goto err_mmio_read_less; 955 } 956 957 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 958 959 /* ENA device administration layer init */ 960 rc = ena_com_admin_init(ena_dev, NULL, true); 961 if (rc) { 962 RTE_LOG(ERR, PMD, 963 "cannot initialize ena admin queue with device\n"); 964 goto err_mmio_read_less; 965 } 966 967 /* To enable the msix interrupts the driver needs to know the number 968 * of queues. So the driver uses polling mode to retrieve this 969 * information. 970 */ 971 ena_com_set_admin_polling_mode(ena_dev, true); 972 973 /* Get Device Attributes and features */ 974 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 975 if (rc) { 976 RTE_LOG(ERR, PMD, 977 "cannot get attribute for ena device rc= %d\n", rc); 978 goto err_admin_init; 979 } 980 981 return 0; 982 983 err_admin_init: 984 ena_com_admin_destroy(ena_dev); 985 986 err_mmio_read_less: 987 ena_com_mmio_reg_read_request_destroy(ena_dev); 988 989 return rc; 990 } 991 992 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 993 { 994 struct rte_pci_device *pci_dev; 995 struct ena_adapter *adapter = 996 (struct ena_adapter *)(eth_dev->data->dev_private); 997 struct ena_com_dev *ena_dev = &adapter->ena_dev; 998 struct ena_com_dev_get_features_ctx get_feat_ctx; 999 int queue_size, rc; 1000 1001 static int adapters_found; 1002 1003 memset(adapter, 0, sizeof(struct ena_adapter)); 1004 ena_dev = &adapter->ena_dev; 1005 1006 eth_dev->dev_ops = &ena_dev_ops; 1007 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1008 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1009 adapter->rte_eth_dev_data = eth_dev->data; 1010 adapter->rte_dev = eth_dev; 1011 1012 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1013 return 0; 1014 1015 pci_dev = eth_dev->pci_dev; 1016 adapter->pdev = pci_dev; 1017 1018 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", 1019 pci_dev->addr.domain, 1020 pci_dev->addr.bus, 1021 pci_dev->addr.devid, 1022 pci_dev->addr.function); 1023 1024 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1025 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1026 1027 /* Present ENA_MEM_BAR indicates available LLQ mode. 1028 * Use corresponding policy 1029 */ 1030 if (adapter->dev_mem_base) 1031 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 1032 else if (adapter->regs) 1033 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1034 else 1035 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", 1036 ENA_REGS_BAR); 1037 1038 ena_dev->reg_bar = adapter->regs; 1039 ena_dev->dmadev = adapter->pdev; 1040 1041 adapter->id_number = adapters_found; 1042 1043 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1044 adapter->id_number); 1045 1046 /* device specific initialization routine */ 1047 rc = ena_device_init(ena_dev, &get_feat_ctx); 1048 if (rc) { 1049 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); 1050 return -1; 1051 } 1052 1053 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1054 if (get_feat_ctx.max_queues.max_llq_num == 0) { 1055 PMD_INIT_LOG(ERR, 1056 "Trying to use LLQ but llq_num is 0.\n" 1057 "Fall back into regular queues.\n"); 1058 ena_dev->tx_mem_queue_type = 1059 ENA_ADMIN_PLACEMENT_POLICY_HOST; 1060 adapter->num_queues = 1061 get_feat_ctx.max_queues.max_sq_num; 1062 } else { 1063 adapter->num_queues = 1064 get_feat_ctx.max_queues.max_llq_num; 1065 } 1066 } else { 1067 adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; 1068 } 1069 1070 queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); 1071 if ((queue_size <= 0) || (adapter->num_queues <= 0)) 1072 return -EFAULT; 1073 1074 adapter->tx_ring_size = queue_size; 1075 adapter->rx_ring_size = queue_size; 1076 1077 /* prepare ring structures */ 1078 ena_init_rings(adapter); 1079 1080 /* Set max MTU for this device */ 1081 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1082 1083 /* Copy MAC address and point DPDK to it */ 1084 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1085 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1086 (struct ether_addr *)adapter->mac_addr); 1087 1088 adapter->drv_stats = rte_zmalloc("adapter stats", 1089 sizeof(*adapter->drv_stats), 1090 RTE_CACHE_LINE_SIZE); 1091 if (!adapter->drv_stats) { 1092 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1093 return -ENOMEM; 1094 } 1095 1096 adapters_found++; 1097 adapter->state = ENA_ADAPTER_STATE_INIT; 1098 1099 return 0; 1100 } 1101 1102 static int ena_dev_configure(struct rte_eth_dev *dev) 1103 { 1104 struct ena_adapter *adapter = 1105 (struct ena_adapter *)(dev->data->dev_private); 1106 1107 if (!(adapter->state == ENA_ADAPTER_STATE_INIT || 1108 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 1109 PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n", 1110 adapter->state); 1111 return -1; 1112 } 1113 1114 switch (adapter->state) { 1115 case ENA_ADAPTER_STATE_INIT: 1116 case ENA_ADAPTER_STATE_STOPPED: 1117 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1118 break; 1119 case ENA_ADAPTER_STATE_CONFIG: 1120 RTE_LOG(WARNING, PMD, 1121 "Ivalid driver state while trying to configure device\n"); 1122 break; 1123 default: 1124 break; 1125 } 1126 1127 return 0; 1128 } 1129 1130 static void ena_init_rings(struct ena_adapter *adapter) 1131 { 1132 int i; 1133 1134 for (i = 0; i < adapter->num_queues; i++) { 1135 struct ena_ring *ring = &adapter->tx_ring[i]; 1136 1137 ring->configured = 0; 1138 ring->type = ENA_RING_TYPE_TX; 1139 ring->adapter = adapter; 1140 ring->id = i; 1141 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1142 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1143 } 1144 1145 for (i = 0; i < adapter->num_queues; i++) { 1146 struct ena_ring *ring = &adapter->rx_ring[i]; 1147 1148 ring->configured = 0; 1149 ring->type = ENA_RING_TYPE_RX; 1150 ring->adapter = adapter; 1151 ring->id = i; 1152 } 1153 } 1154 1155 static void ena_infos_get(struct rte_eth_dev *dev, 1156 struct rte_eth_dev_info *dev_info) 1157 { 1158 struct ena_adapter *adapter; 1159 struct ena_com_dev *ena_dev; 1160 struct ena_com_dev_get_features_ctx feat; 1161 uint32_t rx_feat = 0, tx_feat = 0; 1162 int rc = 0; 1163 1164 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1165 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1166 adapter = (struct ena_adapter *)(dev->data->dev_private); 1167 1168 ena_dev = &adapter->ena_dev; 1169 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1170 1171 dev_info->speed_capa = 1172 ETH_LINK_SPEED_1G | 1173 ETH_LINK_SPEED_2_5G | 1174 ETH_LINK_SPEED_5G | 1175 ETH_LINK_SPEED_10G | 1176 ETH_LINK_SPEED_25G | 1177 ETH_LINK_SPEED_40G | 1178 ETH_LINK_SPEED_50G | 1179 ETH_LINK_SPEED_100G; 1180 1181 /* Get supported features from HW */ 1182 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1183 if (unlikely(rc)) { 1184 RTE_LOG(ERR, PMD, 1185 "Cannot get attribute for ena device rc= %d\n", rc); 1186 return; 1187 } 1188 1189 /* Set Tx & Rx features available for device */ 1190 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1191 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1192 1193 if (feat.offload.tx & 1194 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1195 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1196 DEV_TX_OFFLOAD_UDP_CKSUM | 1197 DEV_TX_OFFLOAD_TCP_CKSUM; 1198 1199 if (feat.offload.tx & 1200 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1201 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1202 DEV_RX_OFFLOAD_UDP_CKSUM | 1203 DEV_RX_OFFLOAD_TCP_CKSUM; 1204 1205 /* Inform framework about available features */ 1206 dev_info->rx_offload_capa = rx_feat; 1207 dev_info->tx_offload_capa = tx_feat; 1208 1209 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1210 dev_info->max_rx_pktlen = adapter->max_mtu; 1211 dev_info->max_mac_addrs = 1; 1212 1213 dev_info->max_rx_queues = adapter->num_queues; 1214 dev_info->max_tx_queues = adapter->num_queues; 1215 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1216 } 1217 1218 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1219 uint16_t nb_pkts) 1220 { 1221 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1222 unsigned int ring_size = rx_ring->ring_size; 1223 unsigned int ring_mask = ring_size - 1; 1224 uint16_t next_to_clean = rx_ring->next_to_clean; 1225 int desc_in_use = 0; 1226 unsigned int recv_idx = 0; 1227 struct rte_mbuf *mbuf = NULL; 1228 struct rte_mbuf *mbuf_head = NULL; 1229 struct rte_mbuf *mbuf_prev = NULL; 1230 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1231 unsigned int completed; 1232 1233 struct ena_com_rx_ctx ena_rx_ctx; 1234 int rc = 0; 1235 1236 /* Check adapter state */ 1237 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1238 RTE_LOG(ALERT, PMD, 1239 "Trying to receive pkts while device is NOT running\n"); 1240 return 0; 1241 } 1242 1243 desc_in_use = ENA_CIRC_COUNT(rx_ring->next_to_use, 1244 next_to_clean, ring_size); 1245 if (unlikely(nb_pkts > desc_in_use)) 1246 nb_pkts = desc_in_use; 1247 1248 for (completed = 0; completed < nb_pkts; completed++) { 1249 int segments = 0; 1250 1251 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1252 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1253 ena_rx_ctx.descs = 0; 1254 /* receive packet context */ 1255 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1256 rx_ring->ena_com_io_sq, 1257 &ena_rx_ctx); 1258 if (unlikely(rc)) { 1259 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1260 return 0; 1261 } 1262 1263 if (unlikely(ena_rx_ctx.descs == 0)) 1264 break; 1265 1266 while (segments < ena_rx_ctx.descs) { 1267 mbuf = rx_buff_info[next_to_clean & ring_mask]; 1268 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1269 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1270 mbuf->refcnt = 1; 1271 mbuf->next = NULL; 1272 if (segments == 0) { 1273 mbuf->nb_segs = ena_rx_ctx.descs; 1274 mbuf->port = rx_ring->port_id; 1275 mbuf->pkt_len = 0; 1276 mbuf_head = mbuf; 1277 } else { 1278 /* for multi-segment pkts create mbuf chain */ 1279 mbuf_prev->next = mbuf; 1280 } 1281 mbuf_head->pkt_len += mbuf->data_len; 1282 1283 mbuf_prev = mbuf; 1284 segments++; 1285 next_to_clean = 1286 ENA_RX_RING_IDX_NEXT(next_to_clean, ring_size); 1287 } 1288 1289 /* fill mbuf attributes if any */ 1290 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1291 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1292 1293 /* pass to DPDK application head mbuf */ 1294 rx_pkts[recv_idx] = mbuf_head; 1295 recv_idx++; 1296 } 1297 1298 /* Burst refill to save doorbells, memory barriers, const interval */ 1299 if (ring_size - desc_in_use - 1 > ENA_RING_DESCS_RATIO(ring_size)) 1300 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use - 1); 1301 1302 rx_ring->next_to_clean = next_to_clean & ring_mask; 1303 1304 return recv_idx; 1305 } 1306 1307 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1308 uint16_t nb_pkts) 1309 { 1310 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1311 unsigned int next_to_use = tx_ring->next_to_use; 1312 struct rte_mbuf *mbuf; 1313 unsigned int ring_size = tx_ring->ring_size; 1314 unsigned int ring_mask = ring_size - 1; 1315 struct ena_com_tx_ctx ena_tx_ctx; 1316 struct ena_tx_buffer *tx_info; 1317 struct ena_com_buf *ebuf; 1318 uint16_t rc, req_id, total_tx_descs = 0; 1319 int sent_idx = 0; 1320 int nb_hw_desc; 1321 1322 /* Check adapter state */ 1323 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1324 RTE_LOG(ALERT, PMD, 1325 "Trying to xmit pkts while device is NOT running\n"); 1326 return 0; 1327 } 1328 1329 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 1330 mbuf = tx_pkts[sent_idx]; 1331 1332 req_id = tx_ring->empty_tx_reqs[next_to_use]; 1333 tx_info = &tx_ring->tx_buffer_info[req_id]; 1334 tx_info->mbuf = mbuf; 1335 tx_info->num_of_bufs = 0; 1336 ebuf = tx_info->bufs; 1337 1338 /* Prepare TX context */ 1339 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1340 memset(&ena_tx_ctx.ena_meta, 0x0, 1341 sizeof(struct ena_com_tx_meta)); 1342 ena_tx_ctx.ena_bufs = ebuf; 1343 ena_tx_ctx.req_id = req_id; 1344 if (tx_ring->tx_mem_queue_type == 1345 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1346 /* prepare the push buffer with 1347 * virtual address of the data 1348 */ 1349 ena_tx_ctx.header_len = 1350 RTE_MIN(mbuf->data_len, 1351 tx_ring->tx_max_header_size); 1352 ena_tx_ctx.push_header = 1353 (void *)((char *)mbuf->buf_addr + 1354 mbuf->data_off); 1355 } /* there's no else as we take advantage of memset zeroing */ 1356 1357 /* Set TX offloads flags, if applicable */ 1358 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); 1359 1360 if (unlikely(mbuf->ol_flags & 1361 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 1362 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 1363 1364 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 1365 1366 /* Process first segment taking into 1367 * consideration pushed header 1368 */ 1369 if (mbuf->data_len > ena_tx_ctx.header_len) { 1370 ebuf->paddr = mbuf->buf_physaddr + 1371 mbuf->data_off + 1372 ena_tx_ctx.header_len; 1373 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 1374 ebuf++; 1375 tx_info->num_of_bufs++; 1376 } 1377 1378 while ((mbuf = mbuf->next) != NULL) { 1379 ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; 1380 ebuf->len = mbuf->data_len; 1381 ebuf++; 1382 tx_info->num_of_bufs++; 1383 } 1384 1385 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1386 1387 /* Write data to device */ 1388 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 1389 &ena_tx_ctx, &nb_hw_desc); 1390 if (unlikely(rc)) 1391 break; 1392 1393 tx_info->tx_descs = nb_hw_desc; 1394 1395 next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, ring_size); 1396 } 1397 1398 /* Let HW do it's best :-) */ 1399 rte_wmb(); 1400 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 1401 1402 /* Clear complete packets */ 1403 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 1404 /* Get Tx info & store how many descs were processed */ 1405 tx_info = &tx_ring->tx_buffer_info[req_id]; 1406 total_tx_descs += tx_info->tx_descs; 1407 1408 /* Free whole mbuf chain */ 1409 mbuf = tx_info->mbuf; 1410 rte_pktmbuf_free(mbuf); 1411 1412 /* Put back descriptor to the ring for reuse */ 1413 tx_ring->empty_tx_reqs[tx_ring->next_to_clean] = req_id; 1414 tx_ring->next_to_clean = 1415 ENA_TX_RING_IDX_NEXT(tx_ring->next_to_clean, 1416 tx_ring->ring_size); 1417 1418 /* If too many descs to clean, leave it for another run */ 1419 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 1420 break; 1421 } 1422 1423 /* acknowledge completion of sent packets */ 1424 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 1425 tx_ring->next_to_use = next_to_use; 1426 return sent_idx; 1427 } 1428 1429 static struct eth_driver rte_ena_pmd = { 1430 { 1431 .name = "rte_ena_pmd", 1432 .id_table = pci_id_ena_map, 1433 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1434 }, 1435 .eth_dev_init = eth_ena_dev_init, 1436 .dev_private_size = sizeof(struct ena_adapter), 1437 }; 1438 1439 static int 1440 rte_ena_pmd_init(const char *name __rte_unused, 1441 const char *params __rte_unused) 1442 { 1443 rte_eth_driver_register(&rte_ena_pmd); 1444 return 0; 1445 }; 1446 1447 struct rte_driver ena_pmd_drv = { 1448 .name = "ena_driver", 1449 .type = PMD_PDEV, 1450 .init = rte_ena_pmd_init, 1451 }; 1452 1453 PMD_REGISTER_DRIVER(ena_pmd_drv); 1454