1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <rte_ether.h> 35 #include <rte_ethdev.h> 36 #include <rte_tcp.h> 37 #include <rte_atomic.h> 38 #include <rte_dev.h> 39 #include <rte_errno.h> 40 41 #include "ena_ethdev.h" 42 #include "ena_logs.h" 43 #include "ena_platform.h" 44 #include "ena_com.h" 45 #include "ena_eth_com.h" 46 47 #include <ena_common_defs.h> 48 #include <ena_regs_defs.h> 49 #include <ena_admin_defs.h> 50 #include <ena_eth_io_defs.h> 51 52 #define ENA_IO_TXQ_IDX(q) (2 * (q)) 53 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) 54 /*reverse version of ENA_IO_RXQ_IDX*/ 55 #define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) 56 57 /* While processing submitted and completed descriptors (rx and tx path 58 * respectively) in a loop it is desired to: 59 * - perform batch submissions while populating sumbissmion queue 60 * - avoid blocking transmission of other packets during cleanup phase 61 * Hence the utilization ratio of 1/8 of a queue size. 62 */ 63 #define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) 64 65 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) 66 #define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) 67 68 #define GET_L4_HDR_LEN(mbuf) \ 69 ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ 70 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) 71 72 #define ENA_RX_RSS_TABLE_LOG_SIZE 7 73 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) 74 #define ENA_HASH_KEY_SIZE 40 75 76 /** Vendor ID used by Amazon devices */ 77 #define PCI_VENDOR_ID_AMAZON 0x1D0F 78 /** Amazon devices */ 79 #define PCI_DEVICE_ID_ENA_VF 0xEC20 80 #define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 81 82 static struct rte_pci_id pci_id_ena_map[] = { 83 #define RTE_PCI_DEV_ID_DECL_ENA(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, 84 85 RTE_PCI_DEV_ID_DECL_ENA(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) 86 RTE_PCI_DEV_ID_DECL_ENA(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) 87 {.device_id = 0}, 88 }; 89 90 static int ena_device_init(struct ena_com_dev *ena_dev, 91 struct ena_com_dev_get_features_ctx *get_feat_ctx); 92 static int ena_dev_configure(struct rte_eth_dev *dev); 93 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 94 uint16_t nb_pkts); 95 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 96 uint16_t nb_desc, unsigned int socket_id, 97 const struct rte_eth_txconf *tx_conf); 98 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 99 uint16_t nb_desc, unsigned int socket_id, 100 const struct rte_eth_rxconf *rx_conf, 101 struct rte_mempool *mp); 102 static uint16_t eth_ena_recv_pkts(void *rx_queue, 103 struct rte_mbuf **rx_pkts, uint16_t nb_pkts); 104 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); 105 static void ena_init_rings(struct ena_adapter *adapter); 106 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 107 static int ena_start(struct rte_eth_dev *dev); 108 static void ena_close(struct rte_eth_dev *dev); 109 static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 110 static void ena_rx_queue_release_all(struct rte_eth_dev *dev); 111 static void ena_tx_queue_release_all(struct rte_eth_dev *dev); 112 static void ena_rx_queue_release(void *queue); 113 static void ena_tx_queue_release(void *queue); 114 static void ena_rx_queue_release_bufs(struct ena_ring *ring); 115 static void ena_tx_queue_release_bufs(struct ena_ring *ring); 116 static int ena_link_update(struct rte_eth_dev *dev, 117 __rte_unused int wait_to_complete); 118 static int ena_queue_restart(struct ena_ring *ring); 119 static int ena_queue_restart_all(struct rte_eth_dev *dev, 120 enum ena_ring_type ring_type); 121 static void ena_stats_restart(struct rte_eth_dev *dev); 122 static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, 123 struct rte_eth_dev_info *dev_info); 124 static int ena_rss_reta_update(struct rte_eth_dev *dev, 125 struct rte_eth_rss_reta_entry64 *reta_conf, 126 uint16_t reta_size); 127 static int ena_rss_reta_query(struct rte_eth_dev *dev, 128 struct rte_eth_rss_reta_entry64 *reta_conf, 129 uint16_t reta_size); 130 131 static struct eth_dev_ops ena_dev_ops = { 132 .dev_configure = ena_dev_configure, 133 .dev_infos_get = ena_infos_get, 134 .rx_queue_setup = ena_rx_queue_setup, 135 .tx_queue_setup = ena_tx_queue_setup, 136 .dev_start = ena_start, 137 .link_update = ena_link_update, 138 .stats_get = ena_stats_get, 139 .mtu_set = ena_mtu_set, 140 .rx_queue_release = ena_rx_queue_release, 141 .tx_queue_release = ena_tx_queue_release, 142 .dev_close = ena_close, 143 .reta_update = ena_rss_reta_update, 144 .reta_query = ena_rss_reta_query, 145 }; 146 147 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, 148 struct ena_com_rx_ctx *ena_rx_ctx) 149 { 150 uint64_t ol_flags = 0; 151 152 if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) 153 ol_flags |= PKT_TX_TCP_CKSUM; 154 else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) 155 ol_flags |= PKT_TX_UDP_CKSUM; 156 157 if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) 158 ol_flags |= PKT_TX_IPV4; 159 else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) 160 ol_flags |= PKT_TX_IPV6; 161 162 if (unlikely(ena_rx_ctx->l4_csum_err)) 163 ol_flags |= PKT_RX_L4_CKSUM_BAD; 164 if (unlikely(ena_rx_ctx->l3_csum_err)) 165 ol_flags |= PKT_RX_IP_CKSUM_BAD; 166 167 mbuf->ol_flags = ol_flags; 168 } 169 170 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, 171 struct ena_com_tx_ctx *ena_tx_ctx) 172 { 173 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; 174 175 if (mbuf->ol_flags & 176 (PKT_TX_L4_MASK | PKT_TX_IP_CKSUM | PKT_TX_TCP_SEG)) { 177 /* check if TSO is required */ 178 if (mbuf->ol_flags & PKT_TX_TCP_SEG) { 179 ena_tx_ctx->tso_enable = true; 180 181 ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf); 182 } 183 184 /* check if L3 checksum is needed */ 185 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) 186 ena_tx_ctx->l3_csum_enable = true; 187 188 if (mbuf->ol_flags & PKT_TX_IPV6) { 189 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6; 190 } else { 191 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4; 192 193 /* set don't fragment (DF) flag */ 194 if (mbuf->packet_type & 195 (RTE_PTYPE_L4_NONFRAG 196 | RTE_PTYPE_INNER_L4_NONFRAG)) 197 ena_tx_ctx->df = true; 198 } 199 200 /* check if L4 checksum is needed */ 201 switch (mbuf->ol_flags & PKT_TX_L4_MASK) { 202 case PKT_TX_TCP_CKSUM: 203 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP; 204 ena_tx_ctx->l4_csum_enable = true; 205 break; 206 case PKT_TX_UDP_CKSUM: 207 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP; 208 ena_tx_ctx->l4_csum_enable = true; 209 break; 210 default: 211 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN; 212 ena_tx_ctx->l4_csum_enable = false; 213 break; 214 } 215 216 ena_meta->mss = mbuf->tso_segsz; 217 ena_meta->l3_hdr_len = mbuf->l3_len; 218 ena_meta->l3_hdr_offset = mbuf->l2_len; 219 /* this param needed only for TSO */ 220 ena_meta->l3_outer_hdr_len = 0; 221 ena_meta->l3_outer_hdr_offset = 0; 222 223 ena_tx_ctx->meta_valid = true; 224 } else { 225 ena_tx_ctx->meta_valid = false; 226 } 227 } 228 229 static void ena_close(struct rte_eth_dev *dev) 230 { 231 struct ena_adapter *adapter = 232 (struct ena_adapter *)(dev->data->dev_private); 233 234 adapter->state = ENA_ADAPTER_STATE_STOPPED; 235 236 ena_rx_queue_release_all(dev); 237 ena_tx_queue_release_all(dev); 238 } 239 240 static int ena_rss_reta_update(struct rte_eth_dev *dev, 241 struct rte_eth_rss_reta_entry64 *reta_conf, 242 uint16_t reta_size) 243 { 244 struct ena_adapter *adapter = 245 (struct ena_adapter *)(dev->data->dev_private); 246 struct ena_com_dev *ena_dev = &adapter->ena_dev; 247 int ret, i; 248 u16 entry_value; 249 int conf_idx; 250 int idx; 251 252 if ((reta_size == 0) || (reta_conf == NULL)) 253 return -EINVAL; 254 255 if (reta_size > ENA_RX_RSS_TABLE_SIZE) { 256 RTE_LOG(WARNING, PMD, 257 "indirection table %d is bigger than supported (%d)\n", 258 reta_size, ENA_RX_RSS_TABLE_SIZE); 259 ret = -EINVAL; 260 goto err; 261 } 262 263 for (i = 0 ; i < reta_size ; i++) { 264 /* each reta_conf is for 64 entries. 265 * to support 128 we use 2 conf of 64 266 */ 267 conf_idx = i / RTE_RETA_GROUP_SIZE; 268 idx = i % RTE_RETA_GROUP_SIZE; 269 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { 270 entry_value = 271 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); 272 ret = ena_com_indirect_table_fill_entry(ena_dev, 273 i, 274 entry_value); 275 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 276 RTE_LOG(ERR, PMD, 277 "Cannot fill indirect table\n"); 278 ret = -ENOTSUP; 279 goto err; 280 } 281 } 282 } 283 284 ret = ena_com_indirect_table_set(ena_dev); 285 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 286 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 287 ret = -ENOTSUP; 288 goto err; 289 } 290 291 RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", 292 __func__, reta_size, adapter->rte_dev->data->port_id); 293 err: 294 return ret; 295 } 296 297 /* Query redirection table. */ 298 static int ena_rss_reta_query(struct rte_eth_dev *dev, 299 struct rte_eth_rss_reta_entry64 *reta_conf, 300 uint16_t reta_size) 301 { 302 struct ena_adapter *adapter = 303 (struct ena_adapter *)(dev->data->dev_private); 304 struct ena_com_dev *ena_dev = &adapter->ena_dev; 305 int ret; 306 int i; 307 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; 308 int reta_conf_idx; 309 int reta_idx; 310 311 if (reta_size == 0 || reta_conf == NULL || 312 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) 313 return -EINVAL; 314 315 ret = ena_com_indirect_table_get(ena_dev, indirect_table); 316 if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { 317 RTE_LOG(ERR, PMD, "cannot get indirect table\n"); 318 ret = -ENOTSUP; 319 goto err; 320 } 321 322 for (i = 0 ; i < reta_size ; i++) { 323 reta_conf_idx = i / RTE_RETA_GROUP_SIZE; 324 reta_idx = i % RTE_RETA_GROUP_SIZE; 325 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) 326 reta_conf[reta_conf_idx].reta[reta_idx] = 327 ENA_IO_RXQ_IDX_REV(indirect_table[i]); 328 } 329 err: 330 return ret; 331 } 332 333 static int ena_rss_init_default(struct ena_adapter *adapter) 334 { 335 struct ena_com_dev *ena_dev = &adapter->ena_dev; 336 uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; 337 int rc, i; 338 u32 val; 339 340 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 341 if (unlikely(rc)) { 342 RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); 343 goto err_rss_init; 344 } 345 346 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 347 val = i % nb_rx_queues; 348 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 349 ENA_IO_RXQ_IDX(val)); 350 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 351 RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); 352 goto err_fill_indir; 353 } 354 } 355 356 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 357 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 358 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 359 RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); 360 goto err_fill_indir; 361 } 362 363 rc = ena_com_set_default_hash_ctrl(ena_dev); 364 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 365 RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); 366 goto err_fill_indir; 367 } 368 369 rc = ena_com_indirect_table_set(ena_dev); 370 if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { 371 RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); 372 goto err_fill_indir; 373 } 374 RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", 375 adapter->rte_dev->data->port_id); 376 377 return 0; 378 379 err_fill_indir: 380 ena_com_rss_destroy(ena_dev); 381 err_rss_init: 382 383 return rc; 384 } 385 386 static void ena_rx_queue_release_all(struct rte_eth_dev *dev) 387 { 388 struct ena_ring **queues = (struct ena_ring **)dev->data->rx_queues; 389 int nb_queues = dev->data->nb_rx_queues; 390 int i; 391 392 for (i = 0; i < nb_queues; i++) 393 ena_rx_queue_release(queues[i]); 394 } 395 396 static void ena_tx_queue_release_all(struct rte_eth_dev *dev) 397 { 398 struct ena_ring **queues = (struct ena_ring **)dev->data->tx_queues; 399 int nb_queues = dev->data->nb_tx_queues; 400 int i; 401 402 for (i = 0; i < nb_queues; i++) 403 ena_tx_queue_release(queues[i]); 404 } 405 406 static void ena_rx_queue_release(void *queue) 407 { 408 struct ena_ring *ring = (struct ena_ring *)queue; 409 struct ena_adapter *adapter = ring->adapter; 410 int ena_qid; 411 412 ena_assert_msg(ring->configured, 413 "API violation - releasing not configured queue"); 414 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 415 "API violation"); 416 417 /* Destroy HW queue */ 418 ena_qid = ENA_IO_RXQ_IDX(ring->id); 419 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 420 421 /* Free all bufs */ 422 ena_rx_queue_release_bufs(ring); 423 424 /* Free ring resources */ 425 if (ring->rx_buffer_info) 426 rte_free(ring->rx_buffer_info); 427 ring->rx_buffer_info = NULL; 428 429 ring->configured = 0; 430 431 RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", 432 ring->port_id, ring->id); 433 } 434 435 static void ena_tx_queue_release(void *queue) 436 { 437 struct ena_ring *ring = (struct ena_ring *)queue; 438 struct ena_adapter *adapter = ring->adapter; 439 int ena_qid; 440 441 ena_assert_msg(ring->configured, 442 "API violation. Releasing not configured queue"); 443 ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, 444 "API violation"); 445 446 /* Destroy HW queue */ 447 ena_qid = ENA_IO_TXQ_IDX(ring->id); 448 ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); 449 450 /* Free all bufs */ 451 ena_tx_queue_release_bufs(ring); 452 453 /* Free ring resources */ 454 if (ring->tx_buffer_info) 455 rte_free(ring->tx_buffer_info); 456 457 if (ring->empty_tx_reqs) 458 rte_free(ring->empty_tx_reqs); 459 460 ring->empty_tx_reqs = NULL; 461 ring->tx_buffer_info = NULL; 462 463 ring->configured = 0; 464 465 RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", 466 ring->port_id, ring->id); 467 } 468 469 static void ena_rx_queue_release_bufs(struct ena_ring *ring) 470 { 471 unsigned int ring_mask = ring->ring_size - 1; 472 473 while (ring->next_to_clean != ring->next_to_use) { 474 struct rte_mbuf *m = 475 ring->rx_buffer_info[ring->next_to_clean & ring_mask]; 476 477 if (m) 478 __rte_mbuf_raw_free(m); 479 480 ring->next_to_clean = 481 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 482 } 483 } 484 485 static void ena_tx_queue_release_bufs(struct ena_ring *ring) 486 { 487 unsigned int ring_mask = ring->ring_size - 1; 488 489 while (ring->next_to_clean != ring->next_to_use) { 490 struct ena_tx_buffer *tx_buf = 491 &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; 492 493 if (tx_buf->mbuf) 494 rte_pktmbuf_free(tx_buf->mbuf); 495 496 ring->next_to_clean = 497 ENA_CIRC_INC(ring->next_to_clean, 1, ring->ring_size); 498 } 499 } 500 501 static int ena_link_update(struct rte_eth_dev *dev, 502 __rte_unused int wait_to_complete) 503 { 504 struct rte_eth_link *link = &dev->data->dev_link; 505 506 link->link_status = 1; 507 link->link_speed = ETH_LINK_SPEED_10G; 508 link->link_duplex = ETH_LINK_FULL_DUPLEX; 509 510 return 0; 511 } 512 513 static int ena_queue_restart_all(struct rte_eth_dev *dev, 514 enum ena_ring_type ring_type) 515 { 516 struct ena_adapter *adapter = 517 (struct ena_adapter *)(dev->data->dev_private); 518 struct ena_ring *queues = NULL; 519 int i = 0; 520 int rc = 0; 521 522 queues = (ring_type == ENA_RING_TYPE_RX) ? 523 adapter->rx_ring : adapter->tx_ring; 524 525 for (i = 0; i < adapter->num_queues; i++) { 526 if (queues[i].configured) { 527 if (ring_type == ENA_RING_TYPE_RX) { 528 ena_assert_msg( 529 dev->data->rx_queues[i] == &queues[i], 530 "Inconsistent state of rx queues\n"); 531 } else { 532 ena_assert_msg( 533 dev->data->tx_queues[i] == &queues[i], 534 "Inconsistent state of tx queues\n"); 535 } 536 537 rc = ena_queue_restart(&queues[i]); 538 539 if (rc) { 540 PMD_INIT_LOG(ERR, 541 "failed to restart queue %d type(%d)\n", 542 i, ring_type); 543 return -1; 544 } 545 } 546 } 547 548 return 0; 549 } 550 551 static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) 552 { 553 uint32_t max_frame_len = adapter->max_mtu; 554 555 if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) 556 max_frame_len = 557 adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; 558 559 return max_frame_len; 560 } 561 562 static int ena_check_valid_conf(struct ena_adapter *adapter) 563 { 564 uint32_t max_frame_len = ena_get_mtu_conf(adapter); 565 566 if (max_frame_len > adapter->max_mtu) { 567 PMD_INIT_LOG(ERR, "Unsupported MTU of %d\n", max_frame_len); 568 return -1; 569 } 570 571 return 0; 572 } 573 574 static int 575 ena_calc_queue_size(struct ena_com_dev *ena_dev, 576 struct ena_com_dev_get_features_ctx *get_feat_ctx) 577 { 578 uint32_t queue_size = ENA_DEFAULT_RING_SIZE; 579 580 queue_size = RTE_MIN(queue_size, 581 get_feat_ctx->max_queues.max_cq_depth); 582 queue_size = RTE_MIN(queue_size, 583 get_feat_ctx->max_queues.max_sq_depth); 584 585 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 586 queue_size = RTE_MIN(queue_size, 587 get_feat_ctx->max_queues.max_llq_depth); 588 589 /* Round down to power of 2 */ 590 if (!rte_is_power_of_2(queue_size)) 591 queue_size = rte_align32pow2(queue_size >> 1); 592 593 if (queue_size == 0) { 594 PMD_INIT_LOG(ERR, "Invalid queue size\n"); 595 return -EFAULT; 596 } 597 598 return queue_size; 599 } 600 601 static void ena_stats_restart(struct rte_eth_dev *dev) 602 { 603 struct ena_adapter *adapter = 604 (struct ena_adapter *)(dev->data->dev_private); 605 606 rte_atomic64_init(&adapter->drv_stats->ierrors); 607 rte_atomic64_init(&adapter->drv_stats->oerrors); 608 rte_atomic64_init(&adapter->drv_stats->imcasts); 609 rte_atomic64_init(&adapter->drv_stats->rx_nombuf); 610 } 611 612 static void ena_stats_get(struct rte_eth_dev *dev, 613 struct rte_eth_stats *stats) 614 { 615 struct ena_admin_basic_stats ena_stats; 616 struct ena_adapter *adapter = 617 (struct ena_adapter *)(dev->data->dev_private); 618 struct ena_com_dev *ena_dev = &adapter->ena_dev; 619 int rc; 620 621 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 622 return; 623 624 memset(&ena_stats, 0, sizeof(ena_stats)); 625 rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); 626 if (unlikely(rc)) { 627 RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); 628 return; 629 } 630 631 /* Set of basic statistics from ENA */ 632 stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high, 633 ena_stats.rx_pkts_low); 634 stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high, 635 ena_stats.tx_pkts_low); 636 stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high, 637 ena_stats.rx_bytes_low); 638 stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, 639 ena_stats.tx_bytes_low); 640 stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, 641 ena_stats.rx_drops_low); 642 643 /* Driver related stats */ 644 stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); 645 stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); 646 stats->imcasts = rte_atomic64_read(&adapter->drv_stats->imcasts); 647 stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); 648 } 649 650 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 651 { 652 struct ena_adapter *adapter; 653 struct ena_com_dev *ena_dev; 654 int rc = 0; 655 656 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 657 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 658 adapter = (struct ena_adapter *)(dev->data->dev_private); 659 660 ena_dev = &adapter->ena_dev; 661 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 662 663 if (mtu > ena_get_mtu_conf(adapter)) { 664 RTE_LOG(ERR, PMD, 665 "Given MTU (%d) exceeds maximum MTU supported (%d)\n", 666 mtu, ena_get_mtu_conf(adapter)); 667 rc = -EINVAL; 668 goto err; 669 } 670 671 rc = ena_com_set_dev_mtu(ena_dev, mtu); 672 if (rc) 673 RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); 674 else 675 RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); 676 677 err: 678 return rc; 679 } 680 681 static int ena_start(struct rte_eth_dev *dev) 682 { 683 struct ena_adapter *adapter = 684 (struct ena_adapter *)(dev->data->dev_private); 685 int rc = 0; 686 687 if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || 688 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 689 PMD_INIT_LOG(ERR, "API violation"); 690 return -1; 691 } 692 693 rc = ena_check_valid_conf(adapter); 694 if (rc) 695 return rc; 696 697 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); 698 if (rc) 699 return rc; 700 701 rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); 702 if (rc) 703 return rc; 704 705 if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & 706 ETH_MQ_RX_RSS_FLAG) { 707 rc = ena_rss_init_default(adapter); 708 if (rc) 709 return rc; 710 } 711 712 ena_stats_restart(dev); 713 714 adapter->state = ENA_ADAPTER_STATE_RUNNING; 715 716 return 0; 717 } 718 719 static int ena_queue_restart(struct ena_ring *ring) 720 { 721 int rc; 722 723 ena_assert_msg(ring->configured == 1, 724 "Trying to restart unconfigured queue\n"); 725 726 ring->next_to_clean = 0; 727 ring->next_to_use = 0; 728 729 if (ring->type == ENA_RING_TYPE_TX) 730 return 0; 731 732 rc = ena_populate_rx_queue(ring, ring->ring_size - 1); 733 if ((unsigned int)rc != ring->ring_size - 1) { 734 PMD_INIT_LOG(ERR, "Failed to populate rx ring !\n"); 735 return (-1); 736 } 737 738 return 0; 739 } 740 741 static int ena_tx_queue_setup(struct rte_eth_dev *dev, 742 uint16_t queue_idx, 743 uint16_t nb_desc, 744 __rte_unused unsigned int socket_id, 745 __rte_unused const struct rte_eth_txconf *tx_conf) 746 { 747 struct ena_ring *txq = NULL; 748 struct ena_adapter *adapter = 749 (struct ena_adapter *)(dev->data->dev_private); 750 unsigned int i; 751 int ena_qid; 752 int rc; 753 struct ena_com_dev *ena_dev = &adapter->ena_dev; 754 755 txq = &adapter->tx_ring[queue_idx]; 756 757 if (txq->configured) { 758 RTE_LOG(CRIT, PMD, 759 "API violation. Queue %d is already configured\n", 760 queue_idx); 761 return -1; 762 } 763 764 if (nb_desc > adapter->tx_ring_size) { 765 RTE_LOG(ERR, PMD, 766 "Unsupported size of TX queue (max size: %d)\n", 767 adapter->tx_ring_size); 768 return -EINVAL; 769 } 770 771 ena_qid = ENA_IO_TXQ_IDX(queue_idx); 772 rc = ena_com_create_io_queue(ena_dev, ena_qid, 773 ENA_COM_IO_QUEUE_DIRECTION_TX, 774 ena_dev->tx_mem_queue_type, 775 -1 /* admin interrupts is not used */, 776 nb_desc); 777 if (rc) { 778 RTE_LOG(ERR, PMD, 779 "failed to create io TX queue #%d (qid:%d) rc: %d\n", 780 queue_idx, ena_qid, rc); 781 } 782 txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 783 txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 784 785 txq->port_id = dev->data->port_id; 786 txq->next_to_clean = 0; 787 txq->next_to_use = 0; 788 txq->ring_size = nb_desc; 789 790 txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", 791 sizeof(struct ena_tx_buffer) * 792 txq->ring_size, 793 RTE_CACHE_LINE_SIZE); 794 if (!txq->tx_buffer_info) { 795 RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); 796 return -ENOMEM; 797 } 798 799 txq->empty_tx_reqs = rte_zmalloc("txq->empty_tx_reqs", 800 sizeof(u16) * txq->ring_size, 801 RTE_CACHE_LINE_SIZE); 802 if (!txq->empty_tx_reqs) { 803 RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); 804 rte_free(txq->tx_buffer_info); 805 return -ENOMEM; 806 } 807 for (i = 0; i < txq->ring_size; i++) 808 txq->empty_tx_reqs[i] = i; 809 810 /* Store pointer to this queue in upper layer */ 811 txq->configured = 1; 812 dev->data->tx_queues[queue_idx] = txq; 813 814 return rc; 815 } 816 817 static int ena_rx_queue_setup(struct rte_eth_dev *dev, 818 uint16_t queue_idx, 819 uint16_t nb_desc, 820 __rte_unused unsigned int socket_id, 821 __rte_unused const struct rte_eth_rxconf *rx_conf, 822 struct rte_mempool *mp) 823 { 824 struct ena_adapter *adapter = 825 (struct ena_adapter *)(dev->data->dev_private); 826 struct ena_ring *rxq = NULL; 827 uint16_t ena_qid = 0; 828 int rc = 0; 829 struct ena_com_dev *ena_dev = &adapter->ena_dev; 830 831 rxq = &adapter->rx_ring[queue_idx]; 832 if (rxq->configured) { 833 RTE_LOG(CRIT, PMD, 834 "API violation. Queue %d is already configured\n", 835 queue_idx); 836 return -1; 837 } 838 839 if (nb_desc > adapter->rx_ring_size) { 840 RTE_LOG(ERR, PMD, 841 "Unsupported size of RX queue (max size: %d)\n", 842 adapter->rx_ring_size); 843 return -EINVAL; 844 } 845 846 ena_qid = ENA_IO_RXQ_IDX(queue_idx); 847 rc = ena_com_create_io_queue(ena_dev, ena_qid, 848 ENA_COM_IO_QUEUE_DIRECTION_RX, 849 ENA_ADMIN_PLACEMENT_POLICY_HOST, 850 -1 /* admin interrupts not used */, 851 nb_desc); 852 if (rc) 853 RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", 854 queue_idx, rc); 855 856 rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; 857 rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; 858 859 rxq->port_id = dev->data->port_id; 860 rxq->next_to_clean = 0; 861 rxq->next_to_use = 0; 862 rxq->ring_size = nb_desc; 863 rxq->mb_pool = mp; 864 865 rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", 866 sizeof(struct rte_mbuf *) * nb_desc, 867 RTE_CACHE_LINE_SIZE); 868 if (!rxq->rx_buffer_info) { 869 RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); 870 return -ENOMEM; 871 } 872 873 /* Store pointer to this queue in upper layer */ 874 rxq->configured = 1; 875 dev->data->rx_queues[queue_idx] = rxq; 876 877 return rc; 878 } 879 880 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) 881 { 882 unsigned int i; 883 int rc; 884 unsigned int ring_size = rxq->ring_size; 885 unsigned int ring_mask = ring_size - 1; 886 int next_to_use = rxq->next_to_use & ring_mask; 887 struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; 888 889 if (unlikely(!count)) 890 return 0; 891 892 ena_assert_msg((((ENA_CIRC_COUNT(rxq->next_to_use, rxq->next_to_clean, 893 rxq->ring_size)) + 894 count) < rxq->ring_size), "bad ring state"); 895 896 count = RTE_MIN(count, ring_size - next_to_use); 897 898 /* get resources for incoming packets */ 899 rc = rte_mempool_get_bulk(rxq->mb_pool, 900 (void **)(&mbufs[next_to_use]), count); 901 if (unlikely(rc < 0)) { 902 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); 903 PMD_RX_LOG(DEBUG, "there are no enough free buffers"); 904 return 0; 905 } 906 907 for (i = 0; i < count; i++) { 908 struct rte_mbuf *mbuf = mbufs[next_to_use]; 909 struct ena_com_buf ebuf; 910 911 rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); 912 /* prepare physical address for DMA transaction */ 913 ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; 914 ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; 915 /* pass resource to device */ 916 rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, 917 &ebuf, next_to_use); 918 if (unlikely(rc)) { 919 RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); 920 break; 921 } 922 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, ring_size); 923 } 924 925 rte_wmb(); 926 rxq->next_to_use = next_to_use; 927 /* let HW know that it can fill buffers with data */ 928 ena_com_write_sq_doorbell(rxq->ena_com_io_sq); 929 930 return i; 931 } 932 933 static int ena_device_init(struct ena_com_dev *ena_dev, 934 struct ena_com_dev_get_features_ctx *get_feat_ctx) 935 { 936 int rc; 937 938 /* Initialize mmio registers */ 939 rc = ena_com_mmio_reg_read_request_init(ena_dev); 940 if (rc) { 941 RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); 942 return rc; 943 } 944 945 /* reset device */ 946 rc = ena_com_dev_reset(ena_dev); 947 if (rc) { 948 RTE_LOG(ERR, PMD, "cannot reset device\n"); 949 goto err_mmio_read_less; 950 } 951 952 /* check FW version */ 953 rc = ena_com_validate_version(ena_dev); 954 if (rc) { 955 RTE_LOG(ERR, PMD, "device version is too low\n"); 956 goto err_mmio_read_less; 957 } 958 959 ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); 960 961 /* ENA device administration layer init */ 962 rc = ena_com_admin_init(ena_dev, NULL, true); 963 if (rc) { 964 RTE_LOG(ERR, PMD, 965 "cannot initialize ena admin queue with device\n"); 966 goto err_mmio_read_less; 967 } 968 969 /* To enable the msix interrupts the driver needs to know the number 970 * of queues. So the driver uses polling mode to retrieve this 971 * information. 972 */ 973 ena_com_set_admin_polling_mode(ena_dev, true); 974 975 /* Get Device Attributes and features */ 976 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 977 if (rc) { 978 RTE_LOG(ERR, PMD, 979 "cannot get attribute for ena device rc= %d\n", rc); 980 goto err_admin_init; 981 } 982 983 return 0; 984 985 err_admin_init: 986 ena_com_admin_destroy(ena_dev); 987 988 err_mmio_read_less: 989 ena_com_mmio_reg_read_request_destroy(ena_dev); 990 991 return rc; 992 } 993 994 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) 995 { 996 struct rte_pci_device *pci_dev; 997 struct ena_adapter *adapter = 998 (struct ena_adapter *)(eth_dev->data->dev_private); 999 struct ena_com_dev *ena_dev = &adapter->ena_dev; 1000 struct ena_com_dev_get_features_ctx get_feat_ctx; 1001 int queue_size, rc; 1002 1003 static int adapters_found; 1004 1005 memset(adapter, 0, sizeof(struct ena_adapter)); 1006 ena_dev = &adapter->ena_dev; 1007 1008 eth_dev->dev_ops = &ena_dev_ops; 1009 eth_dev->rx_pkt_burst = ð_ena_recv_pkts; 1010 eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; 1011 adapter->rte_eth_dev_data = eth_dev->data; 1012 adapter->rte_dev = eth_dev; 1013 1014 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1015 return 0; 1016 1017 pci_dev = eth_dev->pci_dev; 1018 adapter->pdev = pci_dev; 1019 1020 PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", 1021 pci_dev->addr.domain, 1022 pci_dev->addr.bus, 1023 pci_dev->addr.devid, 1024 pci_dev->addr.function); 1025 1026 adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; 1027 adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; 1028 1029 /* Present ENA_MEM_BAR indicates available LLQ mode. 1030 * Use corresponding policy 1031 */ 1032 if (adapter->dev_mem_base) 1033 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; 1034 else if (adapter->regs) 1035 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1036 else 1037 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", 1038 ENA_REGS_BAR); 1039 1040 ena_dev->reg_bar = adapter->regs; 1041 ena_dev->dmadev = adapter->pdev; 1042 1043 adapter->id_number = adapters_found; 1044 1045 snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", 1046 adapter->id_number); 1047 1048 /* device specific initialization routine */ 1049 rc = ena_device_init(ena_dev, &get_feat_ctx); 1050 if (rc) { 1051 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); 1052 return -1; 1053 } 1054 1055 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1056 if (get_feat_ctx.max_queues.max_llq_num == 0) { 1057 PMD_INIT_LOG(ERR, 1058 "Trying to use LLQ but llq_num is 0.\n" 1059 "Fall back into regular queues.\n"); 1060 ena_dev->tx_mem_queue_type = 1061 ENA_ADMIN_PLACEMENT_POLICY_HOST; 1062 adapter->num_queues = 1063 get_feat_ctx.max_queues.max_sq_num; 1064 } else { 1065 adapter->num_queues = 1066 get_feat_ctx.max_queues.max_llq_num; 1067 } 1068 } else { 1069 adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; 1070 } 1071 1072 queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); 1073 if ((queue_size <= 0) || (adapter->num_queues <= 0)) 1074 return -EFAULT; 1075 1076 adapter->tx_ring_size = queue_size; 1077 adapter->rx_ring_size = queue_size; 1078 1079 /* prepare ring structures */ 1080 ena_init_rings(adapter); 1081 1082 /* Set max MTU for this device */ 1083 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 1084 1085 /* Copy MAC address and point DPDK to it */ 1086 eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; 1087 ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, 1088 (struct ether_addr *)adapter->mac_addr); 1089 1090 adapter->drv_stats = rte_zmalloc("adapter stats", 1091 sizeof(*adapter->drv_stats), 1092 RTE_CACHE_LINE_SIZE); 1093 if (!adapter->drv_stats) { 1094 RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); 1095 return -ENOMEM; 1096 } 1097 1098 adapters_found++; 1099 adapter->state = ENA_ADAPTER_STATE_INIT; 1100 1101 return 0; 1102 } 1103 1104 static int ena_dev_configure(struct rte_eth_dev *dev) 1105 { 1106 struct ena_adapter *adapter = 1107 (struct ena_adapter *)(dev->data->dev_private); 1108 1109 if (!(adapter->state == ENA_ADAPTER_STATE_INIT || 1110 adapter->state == ENA_ADAPTER_STATE_STOPPED)) { 1111 PMD_INIT_LOG(ERR, "Illegal adapter state: %d\n", 1112 adapter->state); 1113 return -1; 1114 } 1115 1116 switch (adapter->state) { 1117 case ENA_ADAPTER_STATE_INIT: 1118 case ENA_ADAPTER_STATE_STOPPED: 1119 adapter->state = ENA_ADAPTER_STATE_CONFIG; 1120 break; 1121 case ENA_ADAPTER_STATE_CONFIG: 1122 RTE_LOG(WARNING, PMD, 1123 "Ivalid driver state while trying to configure device\n"); 1124 break; 1125 default: 1126 break; 1127 } 1128 1129 return 0; 1130 } 1131 1132 static void ena_init_rings(struct ena_adapter *adapter) 1133 { 1134 int i; 1135 1136 for (i = 0; i < adapter->num_queues; i++) { 1137 struct ena_ring *ring = &adapter->tx_ring[i]; 1138 1139 ring->configured = 0; 1140 ring->type = ENA_RING_TYPE_TX; 1141 ring->adapter = adapter; 1142 ring->id = i; 1143 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; 1144 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; 1145 } 1146 1147 for (i = 0; i < adapter->num_queues; i++) { 1148 struct ena_ring *ring = &adapter->rx_ring[i]; 1149 1150 ring->configured = 0; 1151 ring->type = ENA_RING_TYPE_RX; 1152 ring->adapter = adapter; 1153 ring->id = i; 1154 } 1155 } 1156 1157 static void ena_infos_get(struct rte_eth_dev *dev, 1158 struct rte_eth_dev_info *dev_info) 1159 { 1160 struct ena_adapter *adapter; 1161 struct ena_com_dev *ena_dev; 1162 struct ena_com_dev_get_features_ctx feat; 1163 uint32_t rx_feat = 0, tx_feat = 0; 1164 int rc = 0; 1165 1166 ena_assert_msg(dev->data != NULL, "Uninitialized device"); 1167 ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); 1168 adapter = (struct ena_adapter *)(dev->data->dev_private); 1169 1170 ena_dev = &adapter->ena_dev; 1171 ena_assert_msg(ena_dev != NULL, "Uninitialized device"); 1172 1173 /* Get supported features from HW */ 1174 rc = ena_com_get_dev_attr_feat(ena_dev, &feat); 1175 if (unlikely(rc)) { 1176 RTE_LOG(ERR, PMD, 1177 "Cannot get attribute for ena device rc= %d\n", rc); 1178 return; 1179 } 1180 1181 /* Set Tx & Rx features available for device */ 1182 if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) 1183 tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; 1184 1185 if (feat.offload.tx & 1186 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) 1187 tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1188 DEV_TX_OFFLOAD_UDP_CKSUM | 1189 DEV_TX_OFFLOAD_TCP_CKSUM; 1190 1191 if (feat.offload.tx & 1192 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) 1193 rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1194 DEV_RX_OFFLOAD_UDP_CKSUM | 1195 DEV_RX_OFFLOAD_TCP_CKSUM; 1196 1197 /* Inform framework about available features */ 1198 dev_info->rx_offload_capa = rx_feat; 1199 dev_info->tx_offload_capa = tx_feat; 1200 1201 dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; 1202 dev_info->max_rx_pktlen = adapter->max_mtu; 1203 dev_info->max_mac_addrs = 1; 1204 1205 dev_info->max_rx_queues = adapter->num_queues; 1206 dev_info->max_tx_queues = adapter->num_queues; 1207 dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; 1208 } 1209 1210 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1211 uint16_t nb_pkts) 1212 { 1213 struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); 1214 unsigned int ring_size = rx_ring->ring_size; 1215 unsigned int ring_mask = ring_size - 1; 1216 uint16_t next_to_clean = rx_ring->next_to_clean; 1217 int desc_in_use = 0; 1218 unsigned int recv_idx = 0; 1219 struct rte_mbuf *mbuf = NULL; 1220 struct rte_mbuf *mbuf_head = NULL; 1221 struct rte_mbuf *mbuf_prev = NULL; 1222 struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; 1223 unsigned int completed; 1224 1225 struct ena_com_rx_ctx ena_rx_ctx; 1226 int rc = 0; 1227 1228 /* Check adapter state */ 1229 if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1230 RTE_LOG(ALERT, PMD, 1231 "Trying to receive pkts while device is NOT running\n"); 1232 return 0; 1233 } 1234 1235 desc_in_use = ENA_CIRC_COUNT(rx_ring->next_to_use, 1236 next_to_clean, ring_size); 1237 if (unlikely(nb_pkts > desc_in_use)) 1238 nb_pkts = desc_in_use; 1239 1240 for (completed = 0; completed < nb_pkts; completed++) { 1241 int segments = 0; 1242 1243 ena_rx_ctx.max_bufs = rx_ring->ring_size; 1244 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; 1245 ena_rx_ctx.descs = 0; 1246 /* receive packet context */ 1247 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, 1248 rx_ring->ena_com_io_sq, 1249 &ena_rx_ctx); 1250 if (unlikely(rc)) { 1251 RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); 1252 return 0; 1253 } 1254 1255 if (unlikely(ena_rx_ctx.descs == 0)) 1256 break; 1257 1258 while (segments < ena_rx_ctx.descs) { 1259 mbuf = rx_buff_info[next_to_clean & ring_mask]; 1260 mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; 1261 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 1262 mbuf->refcnt = 1; 1263 mbuf->next = NULL; 1264 if (segments == 0) { 1265 mbuf->nb_segs = ena_rx_ctx.descs; 1266 mbuf->port = rx_ring->port_id; 1267 mbuf->pkt_len = 0; 1268 mbuf_head = mbuf; 1269 } else { 1270 /* for multi-segment pkts create mbuf chain */ 1271 mbuf_prev->next = mbuf; 1272 } 1273 mbuf_head->pkt_len += mbuf->data_len; 1274 1275 mbuf_prev = mbuf; 1276 segments++; 1277 next_to_clean = 1278 ENA_RX_RING_IDX_NEXT(next_to_clean, ring_size); 1279 } 1280 1281 /* fill mbuf attributes if any */ 1282 ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); 1283 mbuf_head->hash.rss = (uint32_t)rx_ring->id; 1284 1285 /* pass to DPDK application head mbuf */ 1286 rx_pkts[recv_idx] = mbuf_head; 1287 recv_idx++; 1288 } 1289 1290 /* Burst refill to save doorbells, memory barriers, const interval */ 1291 if (ring_size - desc_in_use - 1 > ENA_RING_DESCS_RATIO(ring_size)) 1292 ena_populate_rx_queue(rx_ring, ring_size - desc_in_use - 1); 1293 1294 rx_ring->next_to_clean = next_to_clean & ring_mask; 1295 1296 return recv_idx; 1297 } 1298 1299 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 1300 uint16_t nb_pkts) 1301 { 1302 struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); 1303 unsigned int next_to_use = tx_ring->next_to_use; 1304 struct rte_mbuf *mbuf; 1305 unsigned int ring_size = tx_ring->ring_size; 1306 unsigned int ring_mask = ring_size - 1; 1307 struct ena_com_tx_ctx ena_tx_ctx; 1308 struct ena_tx_buffer *tx_info; 1309 struct ena_com_buf *ebuf; 1310 uint16_t rc, req_id, total_tx_descs = 0; 1311 int sent_idx = 0; 1312 int nb_hw_desc; 1313 1314 /* Check adapter state */ 1315 if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { 1316 RTE_LOG(ALERT, PMD, 1317 "Trying to xmit pkts while device is NOT running\n"); 1318 return 0; 1319 } 1320 1321 for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { 1322 mbuf = tx_pkts[sent_idx]; 1323 1324 req_id = tx_ring->empty_tx_reqs[next_to_use]; 1325 tx_info = &tx_ring->tx_buffer_info[req_id]; 1326 tx_info->mbuf = mbuf; 1327 tx_info->num_of_bufs = 0; 1328 ebuf = tx_info->bufs; 1329 1330 /* Prepare TX context */ 1331 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); 1332 memset(&ena_tx_ctx.ena_meta, 0x0, 1333 sizeof(struct ena_com_tx_meta)); 1334 ena_tx_ctx.ena_bufs = ebuf; 1335 ena_tx_ctx.req_id = req_id; 1336 if (tx_ring->tx_mem_queue_type == 1337 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 1338 /* prepare the push buffer with 1339 * virtual address of the data 1340 */ 1341 ena_tx_ctx.header_len = 1342 RTE_MIN(mbuf->data_len, 1343 tx_ring->tx_max_header_size); 1344 ena_tx_ctx.push_header = 1345 (void *)((char *)mbuf->buf_addr + 1346 mbuf->data_off); 1347 } /* there's no else as we take advantage of memset zeroing */ 1348 1349 /* Set TX offloads flags, if applicable */ 1350 ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); 1351 1352 if (unlikely(mbuf->ol_flags & 1353 (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) 1354 rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); 1355 1356 rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); 1357 1358 /* Process first segment taking into 1359 * consideration pushed header 1360 */ 1361 if (mbuf->data_len > ena_tx_ctx.header_len) { 1362 ebuf->paddr = mbuf->buf_physaddr + 1363 mbuf->data_off + 1364 ena_tx_ctx.header_len; 1365 ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; 1366 ebuf++; 1367 tx_info->num_of_bufs++; 1368 } 1369 1370 while ((mbuf = mbuf->next) != NULL) { 1371 ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; 1372 ebuf->len = mbuf->data_len; 1373 ebuf++; 1374 tx_info->num_of_bufs++; 1375 } 1376 1377 ena_tx_ctx.num_bufs = tx_info->num_of_bufs; 1378 1379 /* Write data to device */ 1380 rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, 1381 &ena_tx_ctx, &nb_hw_desc); 1382 if (unlikely(rc)) 1383 break; 1384 1385 tx_info->tx_descs = nb_hw_desc; 1386 1387 next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use, ring_size); 1388 } 1389 1390 /* Let HW do it's best :-) */ 1391 rte_wmb(); 1392 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); 1393 1394 /* Clear complete packets */ 1395 while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { 1396 /* Get Tx info & store how many descs were processed */ 1397 tx_info = &tx_ring->tx_buffer_info[req_id]; 1398 total_tx_descs += tx_info->tx_descs; 1399 1400 /* Free whole mbuf chain */ 1401 mbuf = tx_info->mbuf; 1402 rte_pktmbuf_free(mbuf); 1403 1404 /* Put back descriptor to the ring for reuse */ 1405 tx_ring->empty_tx_reqs[tx_ring->next_to_clean] = req_id; 1406 tx_ring->next_to_clean = 1407 ENA_TX_RING_IDX_NEXT(tx_ring->next_to_clean, 1408 tx_ring->ring_size); 1409 1410 /* If too many descs to clean, leave it for another run */ 1411 if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) 1412 break; 1413 } 1414 1415 /* acknowledge completion of sent packets */ 1416 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); 1417 tx_ring->next_to_use = next_to_use; 1418 return sent_idx; 1419 } 1420 1421 static struct eth_driver rte_ena_pmd = { 1422 { 1423 .name = "rte_ena_pmd", 1424 .id_table = pci_id_ena_map, 1425 .drv_flags = RTE_PCI_DRV_NEED_MAPPING, 1426 }, 1427 .eth_dev_init = eth_ena_dev_init, 1428 .dev_private_size = sizeof(struct ena_adapter), 1429 }; 1430 1431 static int 1432 rte_ena_pmd_init(const char *name __rte_unused, 1433 const char *params __rte_unused) 1434 { 1435 rte_eth_driver_register(&rte_ena_pmd); 1436 return 0; 1437 }; 1438 1439 struct rte_driver ena_pmd_drv = { 1440 .name = "ena_driver", 1441 .type = PMD_PDEV, 1442 .init = rte_ena_pmd_init, 1443 }; 1444 1445 PMD_REGISTER_DRIVER(ena_pmd_drv); 1446